From 1ff4b36eefcb693c84017ac588c67e792fafd1d7 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Wed, 9 Jul 2014 16:04:40 -0400 Subject: [PATCH] CI: use conda for most build dependencies --- .travis.yml | 16 ++++- ci/install_conda.sh | 104 +++++++++++++++++++++++++++ ci/{install.sh => install_pydata.sh} | 7 +- ci/requirements-2.6.txt | 32 ++++----- ci/requirements-2.7.txt | 50 ++++++------- ci/requirements-2.7_LOCALE.txt | 36 +++++----- ci/requirements-3.3.txt | 34 ++++----- ci/requirements-3.4.txt | 26 +++---- ci/script.sh | 11 +-- 9 files changed, 215 insertions(+), 101 deletions(-) create mode 100755 ci/install_conda.sh rename ci/{install.sh => install_pydata.sh} (97%) diff --git a/.travis.yml b/.travis.yml index d13509805e0f8..a4d323ef8ba95 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python env: + global: # scatterci API key #- secure: "Bx5umgo6WjuGY+5XFa004xjCiX/vq0CyMZ/ETzcs7EIBI1BE/0fIDXOoWhoxbY9HPfdPGlDnDgB9nGqr5wArO2s+BavyKBWg6osZ3dmkfuJPMOWeyCa92EeP+sfKw8e5HSU5MizW9e319wHWOF/xkzdHR7T67Qd5erhv91x4DnQ=" @@ -19,6 +20,7 @@ matrix: - NOSE_ARGS="not slow and not network and not disabled" - CLIPBOARD=xclip - LOCALE_OVERRIDE="it_IT.UTF-8" + - BUILD_TYPE=conda - JOB_NAME: "26_nslow_nnet" - python: 2.7 env: @@ -26,12 +28,14 @@ matrix: - LOCALE_OVERRIDE="zh_CN.GB18030" - FULL_DEPS=true - JOB_TAG=_LOCALE + - BUILD_TYPE=conda - JOB_NAME: "27_slow_nnet_LOCALE" - python: 2.7 env: - NOSE_ARGS="not slow and not disabled" - FULL_DEPS=true - CLIPBOARD_GUI=gtk2 + - BUILD_TYPE=conda - JOB_NAME: "27_nslow" - DOC_BUILD=true # if rst files were changed, build docs in parallel with tests - python: 3.3 @@ -39,18 +43,21 @@ matrix: - NOSE_ARGS="not slow and not disabled" - FULL_DEPS=true - CLIPBOARD=xsel + - BUILD_TYPE=conda - JOB_NAME: "33_nslow" - python: 3.4 env: - NOSE_ARGS="not slow and not disabled" - FULL_DEPS=true - CLIPBOARD=xsel + - BUILD_TYPE=conda - JOB_NAME: "34_nslow" - python: 3.2 env: - NOSE_ARGS="not slow and not disabled" - FULL_DEPS=true - CLIPBOARD_GUI=qt4 + - BUILD_TYPE=pydata - JOB_NAME: "32_nslow" - python: 2.7 env: @@ -59,6 +66,7 @@ matrix: - JOB_NAME: "27_numpy_master" - JOB_TAG=_NUMPY_DEV_master - NUMPY_BUILD=master + - BUILD_TYPE=pydata - PANDAS_TESTING_MODE="deprecate" allow_failures: - python: 3.2 @@ -66,6 +74,7 @@ matrix: - NOSE_ARGS="not slow and not disabled" - FULL_DEPS=true - CLIPBOARD_GUI=qt4 + - BUILD_TYPE=pydata - JOB_NAME: "32_nslow" - python: 2.7 env: @@ -74,11 +83,14 @@ matrix: - JOB_NAME: "27_numpy_master" - JOB_TAG=_NUMPY_DEV_master - NUMPY_BUILD=master + - BUILD_TYPE=pydata - PANDAS_TESTING_MODE="deprecate" before_install: - echo "before_install" - echo $VIRTUAL_ENV + - export PATH="$HOME/miniconda/bin:$PATH" + - sudo apt-get install ccache - df -h - date - pwd @@ -92,7 +104,7 @@ before_install: install: - echo "install" - ci/prep_ccache.sh - - ci/install.sh + - ci/install_${BUILD_TYPE}.sh - ci/submit_ccache.sh before_script: @@ -106,6 +118,6 @@ script: after_script: - if [ -f /tmp/doc.log ]; then cat /tmp/doc.log; fi - - ci/print_versions.py + - source activate pandas && ci/print_versions.py - ci/print_skipped.py /tmp/nosetests.xml - ci/after_script.sh diff --git a/ci/install_conda.sh b/ci/install_conda.sh new file mode 100755 index 0000000000000..ec0aa5fef84ae --- /dev/null +++ b/ci/install_conda.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +# There are 2 distinct pieces that get zipped and cached +# - The venv site-packages dir including the installed dependencies +# - The pandas build artifacts, using the build cache support via +# scripts/use_build_cache.py +# +# if the user opted in to use the cache and we're on a whitelisted fork +# - if the server doesn't hold a cached version of venv/pandas build, +# do things the slow way, and put the results on the cache server +# for the next time. +# - if the cache files are available, instal some necessaries via apt +# (no compiling needed), then directly goto script and collect 200$. +# + +function edit_init() +{ + if [ -n "$LOCALE_OVERRIDE" ]; then + echo "Adding locale to the first line of pandas/__init__.py" + rm -f pandas/__init__.pyc + sedc="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LOCALE_OVERRIDE')\n" + sed -i "$sedc" pandas/__init__.py + echo "head -4 pandas/__init__.py" + head -4 pandas/__init__.py + echo + fi +} + +edit_init + +python_major_version="${TRAVIS_PYTHON_VERSION:0:1}" +[ "$python_major_version" == "2" ] && python_major_version="" + +home_dir=$(pwd) +echo "home_dir: [$home_dir]" + +if [ -n "$LOCALE_OVERRIDE" ]; then + # make sure the locale is available + # probably useless, since you would need to relogin + time sudo locale-gen "$LOCALE_OVERRIDE" +fi + +# Need to enable for locale testing. The location of the locale file(s) is +# distro specific. For example, on Arch Linux all of the locales are in a +# commented file--/etc/locale.gen--that must be commented in to be used +# whereas Ubuntu looks in /var/lib/locales/supported.d/* and generates locales +# based on what's in the files in that folder +time echo 'it_CH.UTF-8 UTF-8' | sudo tee -a /var/lib/locales/supported.d/it +time sudo locale-gen + + +# install gui for clipboard testing +if [ -n "$CLIPBOARD_GUI" ]; then + echo "Using CLIPBOARD_GUI: $CLIPBOARD_GUI" + [ -n "$python_major_version" ] && py="py" + python_cb_gui_pkg=python${python_major_version}-${py}${CLIPBOARD_GUI} + time sudo apt-get $APT_ARGS install $python_cb_gui_pkg +fi + + +# install a clipboard if $CLIPBOARD is not empty +if [ -n "$CLIPBOARD" ]; then + echo "Using clipboard: $CLIPBOARD" + time sudo apt-get $APT_ARGS install $CLIPBOARD +fi + +python_major_version="${TRAVIS_PYTHON_VERSION:0:1}" +[ "$python_major_version" == "2" ] && python_major_version="" + +wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh || exit 1 +bash miniconda.sh -b -p $HOME/miniconda || exit 1 + +conda config --set always_yes yes --set changeps1 no || exit 1 +conda update -q conda || exit 1 +conda config --add channels http://conda.binstar.org/pandas || exit 1 + +# Useful for debugging any issues with conda +conda info -a || exit 1 + +conda create -n pandas python=$TRAVIS_PYTHON_VERSION || exit 1 +conda install -n pandas --file=ci/requirements-${TRAVIS_PYTHON_VERSION}${JOB_TAG}.txt || exit 1 + +conda install -n pandas pip setuptools nose || exit 1 +conda remove -n pandas pandas + +source activate pandas + +# set the compiler cache to work +if [ "$IRON_TOKEN" ]; then + export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH + gcc=$(which gcc) + echo "gcc: $gcc" + ccache=$(which ccache) + echo "ccache: $ccache" + export CC='ccache gcc' +fi + +python setup.py build_ext --inplace && python setup.py develop + +for package in beautifulsoup4 'python-dateutil'; do + pip uninstall --yes $package +done + +true diff --git a/ci/install.sh b/ci/install_pydata.sh similarity index 97% rename from ci/install.sh rename to ci/install_pydata.sh index f146f3ba7ee82..33a6d3854da22 100755 --- a/ci/install.sh +++ b/ci/install_pydata.sh @@ -137,11 +137,8 @@ if [ "$IRON_TOKEN" ]; then fi # build pandas -time python setup.py sdist -pip uninstall cython -y - -# install pandas -time pip install $(find dist | grep gz | head -n 1) +python setup.py build_ext --inplace +python setup.py develop # restore cython (if not numpy building) if [ -z "$NUMPY_BUILD" ]; then diff --git a/ci/requirements-2.6.txt b/ci/requirements-2.6.txt index 3a845f4ee0540..9b338cee26801 100644 --- a/ci/requirements-2.6.txt +++ b/ci/requirements-2.6.txt @@ -1,16 +1,16 @@ -numpy==1.7.0 -cython==0.19.1 -python-dateutil==1.5 -pytz==2013b -http://www.crummy.com/software/BeautifulSoup/bs4/download/4.2/beautifulsoup4-4.2.0.tar.gz -html5lib==1.0b2 -numexpr==1.4.2 -sqlalchemy==0.7.10 -pymysql==0.6.0 -psycopg2==2.5 -scipy==0.11.0 -statsmodels==0.4.3 -xlwt==0.7.5 -openpyxl==2.0.3 -xlsxwriter==0.4.6 -xlrd==0.9.2 +numpy=1.7.0 +cython=0.19.1 +dateutil=1.5 +pytz=2013b +scipy=0.11.0 +xlwt=0.7.5 +xlrd=0.9.2 +openpyxl=2.0.3 +statsmodels=0.4.3 +html5lib=1.0b2 +beautiful-soup=4.2.0 +psycopg2=2.5.1 +numexpr=1.4.2 +pymysql=0.6.0 +sqlalchemy=0.7.8 +xlsxwriter=0.4.6 diff --git a/ci/requirements-2.7.txt b/ci/requirements-2.7.txt index 5b77428a0f6d7..f3df26e7a0c24 100644 --- a/ci/requirements-2.7.txt +++ b/ci/requirements-2.7.txt @@ -1,25 +1,25 @@ -python-dateutil==2.1 -pytz==2013b -xlwt==0.7.5 -numpy==1.8.1 -cython==0.19.1 -bottleneck==0.6.0 -numexpr==2.2.2 -tables==3.0.0 -matplotlib==1.3.1 -openpyxl==1.6.2 -xlsxwriter==0.4.6 -xlrd==0.9.2 -patsy==0.1.0 -sqlalchemy==0.9.6 -pymysql==0.6.1 -psycopg2==2.5.2 -html5lib==1.0b2 -lxml==3.2.1 -scipy==0.13.3 -beautifulsoup4==4.2.1 -statsmodels==0.5.0 -boto==2.26.1 -httplib2==0.8 -python-gflags==2.0 -google-api-python-client==1.2 +dateutil=2.1 +pytz=2013b +xlwt=0.7.5 +numpy=1.7.0 +cython=0.19.1 +numexpr=2.2.2 +pytables=3.0.0 +matplotlib=1.3.1 +openpyxl=1.6.2 +xlrd=0.9.2 +sqlalchemy=0.9.6 +lxml=3.2.1 +scipy +xlsxwriter=0.4.6 +statsmodels +boto=2.26.1 +bottleneck=0.8.0 +psycopg2=2.5.2 +patsy +pymysql=0.6.1 +html5lib=1.0b2 +beautiful-soup=4.2.1 +httplib2=0.8 +python-gflags=2.0 +google-api-python-client=1.2 diff --git a/ci/requirements-2.7_LOCALE.txt b/ci/requirements-2.7_LOCALE.txt index 9af33fe96d58c..036e597e5b788 100644 --- a/ci/requirements-2.7_LOCALE.txt +++ b/ci/requirements-2.7_LOCALE.txt @@ -1,18 +1,18 @@ -python-dateutil -pytz==2013b -xlwt==0.7.5 -openpyxl==1.6.2 -xlsxwriter==0.4.6 -xlrd==0.9.2 -numpy==1.7.1 -cython==0.19.1 -bottleneck==0.6.0 -matplotlib==1.3.0 -patsy==0.1.0 -sqlalchemy==0.8.1 -html5lib==1.0b2 -lxml==3.2.1 -scipy==0.10.0 -beautifulsoup4==4.2.1 -statsmodels==0.4.3 -bigquery==2.0.17 +dateutil +pytz=2013b +xlwt=0.7.5 +openpyxl=1.6.2 +xlsxwriter=0.4.6 +xlrd=0.9.2 +numpy=1.7.1 +cython=0.19.1 +bottleneck=0.8.0 +matplotlib=1.3.0 +patsy=0.1.0 +sqlalchemy=0.8.1 +html5lib=1.0b2 +lxml=3.2.1 +scipy=0.11.0 +beautiful-soup=4.2.1 +statsmodels=0.4.3 +bigquery=2.0.17 diff --git a/ci/requirements-3.3.txt b/ci/requirements-3.3.txt index fc8cb04387a55..c9beec81236fb 100644 --- a/ci/requirements-3.3.txt +++ b/ci/requirements-3.3.txt @@ -1,17 +1,17 @@ -python-dateutil==2.2 -pytz==2013b -openpyxl==1.6.2 -xlsxwriter==0.4.6 -xlrd==0.9.2 -html5lib==1.0b2 -numpy==1.8.0 -cython==0.19.1 -numexpr==2.3 -tables==3.1.0 -bottleneck==0.8.0 -matplotlib==1.2.1 -patsy==0.1.0 -lxml==3.2.1 -scipy==0.13.3 -beautifulsoup4==4.2.1 -statsmodels==0.5.0 +dateutil +pytz=2013b +openpyxl=1.6.2 +xlsxwriter=0.4.6 +xlrd=0.9.2 +html5lib=1.0b2 +numpy=1.8.0 +cython=0.19.1 +numexpr +pytables +bottleneck=0.8.0 +matplotlib +patsy +lxml=3.2.1 +scipy +beautiful-soup=4.2.1 +statsmodels diff --git a/ci/requirements-3.4.txt b/ci/requirements-3.4.txt index 0747e6f54cd73..33d3b3b4dc459 100644 --- a/ci/requirements-3.4.txt +++ b/ci/requirements-3.4.txt @@ -1,19 +1,19 @@ -python-dateutil +dateutil pytz openpyxl xlsxwriter xlrd html5lib -numpy==1.8.0 -cython==0.20.2 -scipy==0.13.3 -numexpr==2.4 -tables==3.1.0 -bottleneck==0.8.0 -matplotlib==1.3.1 patsy -lxml==3.3.5 -sqlalchemy==0.9.6 -pymysql==0.6.1 -psycopg2==2.5.2 -beautifulsoup4 +beautiful-soup +numpy +cython +scipy +numexpr +pytables +matplotlib +lxml +sqlalchemy +bottleneck +pymysql +psycopg2 diff --git a/ci/script.sh b/ci/script.sh index 152a2f1ebdcf9..b1ba7ba79c816 100755 --- a/ci/script.sh +++ b/ci/script.sh @@ -2,22 +2,23 @@ echo "inside $0" +source activate pandas + if [ -n "$LOCALE_OVERRIDE" ]; then export LC_ALL="$LOCALE_OVERRIDE"; echo "Setting LC_ALL to $LOCALE_OVERRIDE" - curdir="$(pwd)" - cd /tmp + pycmd='import pandas; print("pandas detected console encoding: %s" % pandas.get_option("display.encoding"))' python -c "$pycmd" - cd "$curdir" fi # conditionally build and upload docs to GH/pandas-docs/pandas-docs/travis "$TRAVIS_BUILD_DIR"/ci/build_docs.sh 2>&1 > /tmp/doc.log & # doc build log will be shown after tests -echo nosetests --exe -w /tmp -A "$NOSE_ARGS" pandas --with-xunit --xunit-file=/tmp/nosetests.xml -nosetests --exe -w /tmp -A "$NOSE_ARGS" pandas --with-xunit --xunit-file=/tmp/nosetests.xml + +echo nosetests --exe -A "$NOSE_ARGS" pandas --with-xunit --xunit-file=/tmp/nosetests.xml +nosetests --exe -A "$NOSE_ARGS" pandas --with-xunit --xunit-file=/tmp/nosetests.xml RET="$?"