diff --git a/.travis.yml b/.travis.yml index 529f1221899dc..022e11b7db950 100644 --- a/.travis.yml +++ b/.travis.yml @@ -82,15 +82,10 @@ before_install: install: - echo "install start" - ci/prep_cython_cache.sh - - ci/install_travis.sh + - ci/setup_env.sh - ci/submit_cython_cache.sh - echo "install done" -before_script: - - ci/install_db_travis.sh - - export DISPLAY=":99.0" - - ci/before_script_travis.sh - script: - echo "script start" - source activate pandas-dev diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9b1b17b453af3..eee38dadfab90 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -39,9 +39,8 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH sudo apt-get install -y libc6-dev-i386 - ci/incremental/install_miniconda.sh - ci/incremental/setup_conda_environment.sh - displayName: 'Set up environment' + ci/setup_env.sh + displayName: 'Setup environment and build pandas' condition: true # Do not require pandas @@ -59,13 +58,6 @@ jobs: displayName: 'Dependencies consistency' condition: true - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - source activate pandas-dev - ci/incremental/build.sh - displayName: 'Build' - condition: true - # Require pandas - script: | export PATH=$HOME/miniconda3/bin:$PATH diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 7119054cf2f53..f53e284c221c6 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -50,17 +50,9 @@ jobs: steps: - script: | if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386 $EXTRA_APT; fi - echo "Installing Miniconda" - ci/incremental/install_miniconda.sh - export PATH=$HOME/miniconda3/bin:$PATH - echo "Setting up Conda environment" - ci/incremental/setup_conda_environment.sh - displayName: 'Before Install' - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - source activate pandas-dev - ci/incremental/build.sh - displayName: 'Build' + echo "Creating Environment" + ci/setup_env.sh + displayName: 'Setup environment and build pandas' - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev diff --git a/ci/before_script_travis.sh b/ci/before_script_travis.sh deleted file mode 100755 index 0b3939b1906a2..0000000000000 --- a/ci/before_script_travis.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -echo "inside $0" - -if [ "${TRAVIS_OS_NAME}" == "linux" ]; then - sh -e /etc/init.d/xvfb start - sleep 3 -fi - -# Never fail because bad things happened here. -true diff --git a/ci/incremental/build.sh b/ci/incremental/build.sh deleted file mode 100755 index 05648037935a3..0000000000000 --- a/ci/incremental/build.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Make sure any error below is reported as such -set -v -e - -echo "[building extensions]" -python setup.py build_ext -q --inplace -python -m pip install -e . - -echo -echo "[show environment]" -conda list - -echo -echo "[done]" -exit 0 diff --git a/ci/incremental/install_miniconda.sh b/ci/incremental/install_miniconda.sh deleted file mode 100755 index a47dfdb324b34..0000000000000 --- a/ci/incremental/install_miniconda.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -v -e - -# Install Miniconda -unamestr=`uname` -if [[ "$unamestr" == 'Linux' ]]; then - if [[ "$BITS32" == "yes" ]]; then - wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86.sh -O miniconda.sh - else - wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - fi -elif [[ "$unamestr" == 'Darwin' ]]; then - wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh -else - echo Error -fi -chmod +x miniconda.sh -./miniconda.sh -b diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh deleted file mode 100755 index f174c17a614d8..0000000000000 --- a/ci/incremental/setup_conda_environment.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -set -v -e - -CONDA_INSTALL="conda install -q -y" -PIP_INSTALL="pip install -q" - - -# Deactivate any environment -source deactivate -# Display root environment (for debugging) -conda list -# Clean up any left-over from a previous build -# (note workaround for https://github.com/conda/conda/issues/2679: -# `conda env remove` issue) -conda remove --all -q -y -n pandas-dev - -echo -echo "[create env]" -time conda env create -q --file="${ENV_FILE}" || exit 1 - -set +v -source activate pandas-dev -set -v - -# remove any installed pandas package -# w/o removing anything else -echo -echo "[removing installed pandas]" -conda remove pandas -y --force || true -pip uninstall -y pandas || true - -echo -echo "[no installed pandas]" -conda list pandas - -if [ -n "$LOCALE_OVERRIDE" ]; then - sudo locale-gen "$LOCALE_OVERRIDE" -fi - -# # Install the compiler toolchain -# if [[ $(uname) == Linux ]]; then -# if [[ "$CONDA_SUBDIR" == "linux-32" || "$BITS32" == "yes" ]] ; then -# $CONDA_INSTALL gcc_linux-32 gxx_linux-32 -# else -# $CONDA_INSTALL gcc_linux-64 gxx_linux-64 -# fi -# elif [[ $(uname) == Darwin ]]; then -# $CONDA_INSTALL clang_osx-64 clangxx_osx-64 -# # Install llvm-openmp and intel-openmp on OSX too -# $CONDA_INSTALL llvm-openmp intel-openmp -# fi diff --git a/ci/install_db_travis.sh b/ci/install_db_travis.sh deleted file mode 100755 index e4e6d7a5a9b85..0000000000000 --- a/ci/install_db_travis.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -if [ "${TRAVIS_OS_NAME}" != "linux" ]; then - echo "not using dbs on non-linux" - exit 0 -fi - -echo "installing dbs" -mysql -e 'create database pandas_nosetest;' -psql -c 'create database pandas_nosetest;' -U postgres - -echo "done" -exit 0 diff --git a/ci/install_travis.sh b/ci/install_travis.sh deleted file mode 100755 index d1a940f119228..0000000000000 --- a/ci/install_travis.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/bin/bash - -# edit the locale file if needed -function edit_init() -{ - if [ -n "$LOCALE_OVERRIDE" ]; then - echo "[Adding locale to the first line of pandas/__init__.py]" - rm -f pandas/__init__.pyc - sedc="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LOCALE_OVERRIDE')\n" - sed -i "$sedc" pandas/__init__.py - echo "[head -4 pandas/__init__.py]" - head -4 pandas/__init__.py - echo - fi -} - -echo -echo "[install_travis]" -edit_init - -home_dir=$(pwd) -echo -echo "[home_dir]: $home_dir" - -# install miniconda -MINICONDA_DIR="$HOME/miniconda3" - -echo -echo "[Using clean Miniconda install]" - -if [ -d "$MINICONDA_DIR" ]; then - rm -rf "$MINICONDA_DIR" -fi - -# install miniconda -if [ "${TRAVIS_OS_NAME}" == "osx" ]; then - time wget http://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -q -O miniconda.sh || exit 1 -else - time wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -q -O miniconda.sh || exit 1 -fi -time bash miniconda.sh -b -p "$MINICONDA_DIR" || exit 1 - -echo -echo "[show conda]" -which conda - -echo -echo "[update conda]" -conda config --set ssl_verify false || exit 1 -conda config --set quiet true --set always_yes true --set changeps1 false || exit 1 -conda update -q conda - -# Useful for debugging any issues with conda -conda info -a || exit 1 - -# set the compiler cache to work -echo -if [ -z "$NOCACHE" ] && [ "${TRAVIS_OS_NAME}" == "linux" ]; then - echo "[Using ccache]" - export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH - gcc=$(which gcc) - echo "[gcc]: $gcc" - ccache=$(which ccache) - echo "[ccache]: $ccache" - export CC='ccache gcc' -elif [ -z "$NOCACHE" ] && [ "${TRAVIS_OS_NAME}" == "osx" ]; then - echo "[Install ccache]" - brew install ccache > /dev/null 2>&1 - echo "[Using ccache]" - export PATH=/usr/local/opt/ccache/libexec:$PATH - gcc=$(which gcc) - echo "[gcc]: $gcc" - ccache=$(which ccache) - echo "[ccache]: $ccache" -else - echo "[Not using ccache]" -fi - -echo -echo "[create env]" - -# create our environment -time conda env create -q --file="${ENV_FILE}" || exit 1 - -source activate pandas-dev - -# remove any installed pandas package -# w/o removing anything else -echo -echo "[removing installed pandas]" -conda remove pandas -y --force -pip uninstall -y pandas - -echo -echo "[no installed pandas]" -conda list pandas -pip list --format columns |grep pandas - -# build and install -echo "[running setup.py develop]" -python setup.py develop || exit 1 - -echo -echo "[show environment]" -conda list - -echo -echo "[done]" -exit 0 diff --git a/ci/setup_env.sh b/ci/setup_env.sh new file mode 100755 index 0000000000000..414a5c8705ee9 --- /dev/null +++ b/ci/setup_env.sh @@ -0,0 +1,135 @@ +#!/bin/bash -e + + +# edit the locale file if needed +if [ -n "$LOCALE_OVERRIDE" ]; then + echo "Adding locale to the first line of pandas/__init__.py" + rm -f pandas/__init__.pyc + SEDC="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LOCALE_OVERRIDE')\n" + sed -i "$SEDC" pandas/__init__.py + echo "[head -4 pandas/__init__.py]" + head -4 pandas/__init__.py + echo + sudo locale-gen "$LOCALE_OVERRIDE" +fi + +MINICONDA_DIR="$HOME/miniconda3" + + +if [ -d "$MINICONDA_DIR" ]; then + echo + echo "rm -rf "$MINICONDA_DIR"" + rm -rf "$MINICONDA_DIR" +fi + +echo "Install Miniconda" +UNAME_OS=$(uname) +if [[ "$UNAME_OS" == 'Linux' ]]; then + if [[ "$BITS32" == "yes" ]]; then + CONDA_OS="Linux-x86" + else + CONDA_OS="Linux-x86_64" + fi +elif [[ "$UNAME_OS" == 'Darwin' ]]; then + CONDA_OS="MacOSX-x86_64" +else + echo "OS $UNAME_OS not supported" + exit 1 +fi + +wget -q "https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh" -O miniconda.sh +chmod +x miniconda.sh +./miniconda.sh -b + +export PATH=$MINICONDA_DIR/bin:$PATH + +echo +echo "which conda" +which conda + +echo +echo "update conda" +conda config --set ssl_verify false +conda config --set quiet true --set always_yes true --set changeps1 false +conda update -n base conda + +echo "conda info -a" +conda info -a + +echo +echo "set the compiler cache to work" +if [ -z "$NOCACHE" ] && [ "${TRAVIS_OS_NAME}" == "linux" ]; then + echo "Using ccache" + export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH + GCC=$(which gcc) + echo "gcc: $GCC" + CCACHE=$(which ccache) + echo "ccache: $CCACHE" + export CC='ccache gcc' +elif [ -z "$NOCACHE" ] && [ "${TRAVIS_OS_NAME}" == "osx" ]; then + echo "Install ccache" + brew install ccache > /dev/null 2>&1 + echo "Using ccache" + export PATH=/usr/local/opt/ccache/libexec:$PATH + gcc=$(which gcc) + echo "gcc: $gcc" + CCACHE=$(which ccache) + echo "ccache: $CCACHE" +else + echo "Not using ccache" +fi + +echo "source deactivate" +source deactivate + +echo "conda list (root environment)" +conda list + +# Clean up any left-over from a previous build +# (note workaround for https://github.com/conda/conda/issues/2679: +# `conda env remove` issue) +conda remove --all -q -y -n pandas-dev + +echo +echo "conda env create -q --file=${ENV_FILE}" +time conda env create -q --file="${ENV_FILE}" + +echo "activate pandas-dev" +source activate pandas-dev + +echo +echo "remove any installed pandas package" +echo "w/o removing anything else" +conda remove pandas -y --force || true +pip uninstall -y pandas || true + +echo +echo "conda list pandas" +conda list pandas + +# Make sure any error below is reported as such + +echo "Build extensions and install pandas" +python setup.py build_ext -q --inplace +python -m pip install -e . + +echo +echo "conda list" +conda list + +# Install DB for Linux +export DISPLAY=":99." +if [ ${TRAVIS_OS_NAME} == "linux" ]; then + echo "installing dbs" + mysql -e 'create database pandas_nosetest;' + psql -c 'create database pandas_nosetest;' -U postgres + + echo + echo "sh -e /etc/init.d/xvfb start" + sh -e /etc/init.d/xvfb start + sleep 3 +else + echo "not using dbs on non-linux" +fi + +echo "done"