From d45b6fb441a550a67361460453e28a6bd66c1863 Mon Sep 17 00:00:00 2001 From: "azure-pipelines[bot]" Date: Wed, 19 Sep 2018 11:14:10 +0000 Subject: [PATCH 01/14] Set up CI with Azure Pipelines --- azure-pipelines.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000000000..098e6dbf150ab --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,25 @@ +# Adapted from https://github.com/numba/numba/blob/master/azure-pipelines.yml +jobs: +# Mac and Linux could potentially use the same template +# except it isn't clear how to use a different build matrix +# for each, so for now they are separate +- template: ci/azure/macos.yml + parameters: + name: macOS + vmImage: xcode9-macos10.13 +- template: ci/azure/linux.yml + parameters: + name: Linux + vmImage: ubuntu-16.04 + +# Windows Python 2.7 needs VC 9.0 installed, and not sure +# how to make that a conditional task, so for now these are +# separate templates as well +- template: ci/azure/windows.yml + parameters: + name: Windows + vmImage: vs2017-win2017 +- template: ci/azure/windows-py27.yml + parameters: + name: WindowsPy27 +vmImage: vs2017-win2017 \ No newline at end of file From 7f8866d439493c2713e1240f4acbf22534b77bac Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 19 Sep 2018 06:32:28 -0500 Subject: [PATCH 02/14] Remove appveyor (temporarily) --- appveyor.yml | 91 ---------------------------------------------------- 1 file changed, 91 deletions(-) delete mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index c6199c1493f22..0000000000000 --- a/appveyor.yml +++ /dev/null @@ -1,91 +0,0 @@ -# With infos from -# http://tjelvarolsson.com/blog/how-to-continuously-test-your-python-code-on-windows-using-appveyor/ -# https://packaging.python.org/en/latest/appveyor/ -# https://github.com/rmcgibbo/python-appveyor-conda-example - -# Backslashes in quotes need to be escaped: \ -> "\\" - -matrix: - fast_finish: true # immediately finish build once one of the jobs fails. - -environment: - global: - # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the - # /E:ON and /V:ON options are not enabled in the batch script interpreter - # See: http://stackoverflow.com/a/13751649/163740 - CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci\\run_with_env.cmd" - clone_folder: C:\projects\pandas - PANDAS_TESTING_MODE: "deprecate" - - matrix: - - - CONDA_ROOT: "C:\\Miniconda3_64" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - PYTHON_VERSION: "3.6" - PYTHON_ARCH: "64" - CONDA_PY: "36" - CONDA_NPY: "113" - - - CONDA_ROOT: "C:\\Miniconda3_64" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - PYTHON_VERSION: "2.7" - PYTHON_ARCH: "64" - CONDA_PY: "27" - CONDA_NPY: "110" - -# We always use a 64-bit machine, but can build x86 distributions -# with the PYTHON_ARCH variable (which is used by CMD_IN_ENV). -platform: - - x64 - -# all our python builds have to happen in tests_script... -build: false - -install: - # cancel older builds for the same PR - - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` - https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` - Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` - throw "There are newer queued builds for this pull request, failing early." } - - # this installs the appropriate Miniconda (Py2/Py3, 32/64 bit) - # updates conda & installs: conda-build jinja2 anaconda-client - - powershell .\ci\install.ps1 - - SET PATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%PATH% - - echo "install" - - cd - - ls -ltr - - git tag --sort v:refname - - # this can conflict with git - - cmd: rmdir C:\cygwin /s /q - - # install our build environment - - cmd: conda config --set show_channel_urls true --set always_yes true --set changeps1 false - - cmd: conda update -q conda - - cmd: conda config --set ssl_verify false - - # add the pandas channel *before* defaults to have defaults take priority - - cmd: conda config --add channels conda-forge - - cmd: conda config --add channels pandas - - cmd: conda config --remove channels defaults - - cmd: conda config --add channels defaults - - # this is now the downloaded conda... - - cmd: conda info -a - - # create our env - - cmd: conda env create -q -n pandas --file=ci\appveyor-%CONDA_PY%.yaml - - cmd: activate pandas - - cmd: conda list -n pandas - # uninstall pandas if it's present - - cmd: conda remove pandas -y --force & exit 0 - - cmd: pip uninstall -y pandas & exit 0 - - # build em using the local source checkout in the correct windows env - - cmd: '%CMD_IN_ENV% python setup.py build_ext --inplace' - -test_script: - # tests - - cmd: activate pandas - - cmd: test.bat From 4d59634c111b28c60bd03e38c09327a708acfe71 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 19 Sep 2018 06:32:43 -0500 Subject: [PATCH 03/14] Try windows --- ci/azure-27.yaml | 31 +++++++++++++++++++++ ci/azure-36.yaml | 28 +++++++++++++++++++ ci/azure/windows-py27.yml | 32 ++++++++++++++++++++++ ci/azure/windows.yml | 32 ++++++++++++++++++++++ ci/incremental/build.cmd | 10 +++++++ ci/incremental/setup_conda_environment.cmd | 21 ++++++++++++++ 6 files changed, 154 insertions(+) create mode 100644 ci/azure-27.yaml create mode 100644 ci/azure-36.yaml create mode 100644 ci/azure/windows-py27.yml create mode 100644 ci/azure/windows.yml create mode 100644 ci/incremental/build.cmd create mode 100644 ci/incremental/setup_conda_environment.cmd diff --git a/ci/azure-27.yaml b/ci/azure-27.yaml new file mode 100644 index 0000000000000..bcd9ddee1715e --- /dev/null +++ b/ci/azure-27.yaml @@ -0,0 +1,31 @@ +name: pandas +channels: + - defaults + - conda-forge +dependencies: + - beautifulsoup4 + - bottleneck + - dateutil + - gcsfs + - html5lib + - jinja2=2.8 + - lxml + - matplotlib + - numexpr + - numpy=1.12* + - openpyxl=2.5.5 + - pytables + - python=2.7.* + - pytz + - s3fs + - scipy + - sqlalchemy + - xlrd + - xlsxwriter + - xlwt + # universal + - cython>=0.28.2 + - pytest + - pytest-xdist + - moto + - hypothesis>=3.58.0 diff --git a/ci/azure-36.yaml b/ci/azure-36.yaml new file mode 100644 index 0000000000000..6230e9b6a1885 --- /dev/null +++ b/ci/azure-36.yaml @@ -0,0 +1,28 @@ +name: pandas +channels: + - defaults + - conda-forge +dependencies: + - blosc + - bottleneck + - fastparquet + - feather-format + - matplotlib + - numexpr + - numpy=1.14* + - openpyxl=2.5.5 + - pyarrow + - pytables + - python-dateutil + - python=3.6.* + - pytz + - scipy + - thrift=0.10* + - xlrd + - xlsxwriter + - xlwt + # universal + - cython>=0.28.2 + - pytest + - pytest-xdist + - hypothesis>=3.58.0 diff --git a/ci/azure/windows-py27.yml b/ci/azure/windows-py27.yml new file mode 100644 index 0000000000000..ce53bfc41629f --- /dev/null +++ b/ci/azure/windows-py27.yml @@ -0,0 +1,32 @@ +parameters: + name: '' + vmImage: '' + +jobs: +- job: ${{ parameters.name }} + pool: + vmImage: ${{ parameters.vmImage }} + strategy: + maxParallel: 11 + matrix: + py36_np14: + ENV_FILE: ci/azure-27.yml + CONDA_PY: "27" + CONDA_ENV: pandas + + steps: + - task: CondaEnvironment@1 + inputs: + updateConda: no + packageSpecs: '' + + - script: | + ci\\incremental\\setup_conda_environment.cmd + displayName: 'Before Install' + - script: | + ci\\incremental\\build.cmd + displayName: 'Build' + - script: | + call activate %CONDA_ENV% + pytest --skip-slow --skip-network pandas -n 2 -r sxX --strict %* + displayName: 'Test' diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml new file mode 100644 index 0000000000000..de57d244c38cf --- /dev/null +++ b/ci/azure/windows.yml @@ -0,0 +1,32 @@ +parameters: + name: '' + vmImage: '' + +jobs: +- job: ${{ parameters.name }} + pool: + vmImage: ${{ parameters.vmImage }} + strategy: + maxParallel: 11 + matrix: + py36_np14: + ENV_FILE: ci/azure-36.yml + CONDA_PY: "36" + CONDA_ENV: pandas + + steps: + - task: CondaEnvironment@1 + inputs: + updateConda: no + packageSpecs: '' + + - script: | + ci\\incremental\\setup_conda_environment.cmd + displayName: 'Before Install' + - script: | + ci\\incremental\\build.cmd + displayName: 'Build' + - script: | + call activate %CONDA_ENV% + pytest --skip-slow --skip-network pandas -n 2 -r sxX --strict %* + displayName: 'Test' diff --git a/ci/incremental/build.cmd b/ci/incremental/build.cmd new file mode 100644 index 0000000000000..d2fd06d7d9e50 --- /dev/null +++ b/ci/incremental/build.cmd @@ -0,0 +1,10 @@ +@rem https://github.com/numba/numba/blob/master/buildscripts/incremental/build.cmd +call activate %CONDA_ENV% + +@rem Build numba extensions without silencing compile errors +python setup.py build_ext -q --inplace + +@rem Install pandas locally +python -m pip install -e . + +if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/ci/incremental/setup_conda_environment.cmd b/ci/incremental/setup_conda_environment.cmd new file mode 100644 index 0000000000000..c3c543e2a0d68 --- /dev/null +++ b/ci/incremental/setup_conda_environment.cmd @@ -0,0 +1,21 @@ +@rem https://github.com/numba/numba/blob/master/buildscripts/incremental/setup_conda_environment.cmd +@rem The cmd /C hack circumvents a regression where conda installs a conda.bat +@rem script in non-root environments. +set CONDA_INSTALL=cmd /C conda install -q -y +set PIP_INSTALL=pip install -q + +@echo on + +@rem Deactivate any environment +call deactivate +@rem Display root environment (for debugging) +conda list +@rem Clean up any left-over from a previous build +conda remove --all -q -y -n %CONDA_ENV% +@rem Scipy, CFFI, jinja2 and IPython are optional dependencies, but exercised in the test suite +conda env create -n %CONDA_ENV% --file=ci\azure-%CONDA_PY%.yaml + +call activate %CONDA_ENV% +conda list + +if %errorlevel% neq 0 exit /b %errorlevel% From 9c7f2338082fd56b5558852b3ce3180f8025496e Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 19 Sep 2018 06:35:03 -0500 Subject: [PATCH 04/14] indendation --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 098e6dbf150ab..0523f798d249e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,4 +22,4 @@ jobs: - template: ci/azure/windows-py27.yml parameters: name: WindowsPy27 -vmImage: vs2017-win2017 \ No newline at end of file + vmImage: vs2017-win2017 From 5d0ede68dc549a3ea47d13888f76d2ee50cb0112 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 19 Sep 2018 06:37:29 -0500 Subject: [PATCH 05/14] Just windows for now --- azure-pipelines.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0523f798d249e..3eec92d936a43 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,14 +3,14 @@ jobs: # Mac and Linux could potentially use the same template # except it isn't clear how to use a different build matrix # for each, so for now they are separate -- template: ci/azure/macos.yml - parameters: - name: macOS - vmImage: xcode9-macos10.13 -- template: ci/azure/linux.yml - parameters: - name: Linux - vmImage: ubuntu-16.04 +# - template: ci/azure/macos.yml +# parameters: +# name: macOS +# vmImage: xcode9-macos10.13 +# - template: ci/azure/linux.yml +# parameters: +# name: Linux +# vmImage: ubuntu-16.04 # Windows Python 2.7 needs VC 9.0 installed, and not sure # how to make that a conditional task, so for now these are From 78e3282c4c05f5d81d55c6e2b7cd554c857f6136 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 19 Sep 2018 06:56:22 -0500 Subject: [PATCH 06/14] Add macos --- azure-pipelines.yml | 8 ++-- ci/azure-macos-35.yml | 28 ++++++++++++ ci/{azure-27.yaml => azure-windows-27.yaml} | 0 ci/{azure-36.yaml => azure-windows-36.yaml} | 0 ci/azure/macos.yml | 39 ++++++++++++++++ ci/azure/windows-py27.yml | 2 +- ci/azure/windows.yml | 2 +- ci/incremental/build.sh | 18 ++++++++ ci/incremental/install_miniconda.sh | 19 ++++++++ ci/incremental/setup_conda_environment.sh | 49 +++++++++++++++++++++ 10 files changed, 159 insertions(+), 6 deletions(-) create mode 100644 ci/azure-macos-35.yml rename ci/{azure-27.yaml => azure-windows-27.yaml} (100%) rename ci/{azure-36.yaml => azure-windows-36.yaml} (100%) create mode 100644 ci/azure/macos.yml create mode 100644 ci/incremental/build.sh create mode 100644 ci/incremental/install_miniconda.sh create mode 100644 ci/incremental/setup_conda_environment.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3eec92d936a43..c82dafa224961 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,10 +3,10 @@ jobs: # Mac and Linux could potentially use the same template # except it isn't clear how to use a different build matrix # for each, so for now they are separate -# - template: ci/azure/macos.yml -# parameters: -# name: macOS -# vmImage: xcode9-macos10.13 +- template: ci/azure/macos.yml + parameters: + name: macOS + vmImage: xcode9-macos10.13 # - template: ci/azure/linux.yml # parameters: # name: Linux diff --git a/ci/azure-macos-35.yml b/ci/azure-macos-35.yml new file mode 100644 index 0000000000000..a36f748ded812 --- /dev/null +++ b/ci/azure-macos-35.yml @@ -0,0 +1,28 @@ +name: pandas +channels: + - defaults +dependencies: + - beautifulsoup4 + - bottleneck + - cython>=0.28.2 + - html5lib + - jinja2 + - lxml + - matplotlib + - nomkl + - numexpr + - numpy=1.10.4 + - openpyxl=2.5.5 + - pytables + - python=3.5* + - pytz + - xarray + - xlrd + - xlsxwriter + - xlwt + # universal + - pytest + - pytest-xdist + - pip: + - python-dateutil==2.5.3 + - hypothesis>=3.58.0 diff --git a/ci/azure-27.yaml b/ci/azure-windows-27.yaml similarity index 100% rename from ci/azure-27.yaml rename to ci/azure-windows-27.yaml diff --git a/ci/azure-36.yaml b/ci/azure-windows-36.yaml similarity index 100% rename from ci/azure-36.yaml rename to ci/azure-windows-36.yaml diff --git a/ci/azure/macos.yml b/ci/azure/macos.yml new file mode 100644 index 0000000000000..802b3722e3e27 --- /dev/null +++ b/ci/azure/macos.yml @@ -0,0 +1,39 @@ +parameters: + name: '' + vmImage: '' + +jobs: +- job: ${{ parameters.name }} + pool: + vmImage: ${{ parameters.vmImage }} + strategy: + maxParallel: 11 + matrix: + py35_np_110: + ENV_FILE: ci/azure-macos-35.yml + CONDA_PY: "35" + CONDA_ENV: pandas + TEST_ARGS: "--skip-slow --skip-network" + + steps: + - script: | + if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386; fi + echo "Installing Miniconda" + ci/incremental/install_miniconda.sh + export PATH=$HOME/miniconda3/bin:$PATH + echo "Setting up Conda environment" + ci/incremental/setup_conda_environment.sh + displayName: 'Before Install' + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/incremental/build.sh + displayName: 'Build' + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/incremental/script_single.sh + ci/incremental/script_multi.sh + echo "Test done" + displayName: 'Test' + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd diff --git a/ci/azure/windows-py27.yml b/ci/azure/windows-py27.yml index ce53bfc41629f..760310f88752e 100644 --- a/ci/azure/windows-py27.yml +++ b/ci/azure/windows-py27.yml @@ -10,7 +10,7 @@ jobs: maxParallel: 11 matrix: py36_np14: - ENV_FILE: ci/azure-27.yml + ENV_FILE: ci/azure-windows-27.yml CONDA_PY: "27" CONDA_ENV: pandas diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index de57d244c38cf..6090139fb4f3e 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -10,7 +10,7 @@ jobs: maxParallel: 11 matrix: py36_np14: - ENV_FILE: ci/azure-36.yml + ENV_FILE: ci/azure-windows-36.yml CONDA_PY: "36" CONDA_ENV: pandas diff --git a/ci/incremental/build.sh b/ci/incremental/build.sh new file mode 100644 index 0000000000000..f5b5ecb5ab690 --- /dev/null +++ b/ci/incremental/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +source activate $CONDA_ENV + +# Make sure any error below is reported as such +set -v -e + +echo "[building extensions]" +python setup.py build_ext -q --inplace +python -m pip install -e + +echo +echo "[show environment]" +conda list + +echo +echo "[done]" +exit 0 diff --git a/ci/incremental/install_miniconda.sh b/ci/incremental/install_miniconda.sh new file mode 100644 index 0000000000000..4aa73bac6729b --- /dev/null +++ b/ci/incremental/install_miniconda.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -v -e + +# Install Miniconda +unamestr=`uname` +if [[ "$unamestr" == 'Linux' ]]; then + if [[ "$BITS32" == "yes" ]]; then + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86.sh -O miniconda.sh + else + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + fi +elif [[ "$unamestr" == 'Darwin' ]]; then + wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh +else + echo Error +fi +chmod +x miniconda.sh +./miniconda.sh -b diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh new file mode 100644 index 0000000000000..c5fa701f99ecd --- /dev/null +++ b/ci/incremental/setup_conda_environment.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -v -e + +CONDA_INSTALL="conda install -q -y" +PIP_INSTALL="pip install -q" + +# Deactivate any environment +source deactivate +# Display root environment (for debugging) +conda list +# Clean up any left-over from a previous build +# (note workaround for https://github.com/conda/conda/issues/2679: +# `conda env remove` issue) +conda remove --all -q -y -n $CONDA_ENV + +echo +echo "[create env]" +time conda env create -q -n "${CONDA_ENV}" --file="${ENV_FILE}" || exit 1 + +# Activate first +set +v +source activate $CONDA_ENV +set -v + +# remove any installed pandas package +# w/o removing anything else +echo +echo "[removing installed pandas]" +conda remove pandas -y --force +pip uninstall -y pandas + +echo +echo "[no installed pandas]" +conda list pandas +pip list --format columns |grep pandas + +# # Install the compiler toolchain +# if [[ $(uname) == Linux ]]; then +# if [[ "$CONDA_SUBDIR" == "linux-32" || "$BITS32" == "yes" ]] ; then +# $CONDA_INSTALL gcc_linux-32 gxx_linux-32 +# else +# $CONDA_INSTALL gcc_linux-64 gxx_linux-64 +# fi +# elif [[ $(uname) == Darwin ]]; then +# $CONDA_INSTALL clang_osx-64 clangxx_osx-64 +# # Install llvm-openmp and intel-openmp on OSX too +# $CONDA_INSTALL llvm-openmp intel-openmp +# fi From bf507c6f51f67e724154c5afefc982b1ae42dd73 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 19 Sep 2018 07:57:31 -0500 Subject: [PATCH 07/14] Path to azure env --- ci/incremental/setup_conda_environment.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/incremental/setup_conda_environment.cmd b/ci/incremental/setup_conda_environment.cmd index c3c543e2a0d68..b4446c49fabd3 100644 --- a/ci/incremental/setup_conda_environment.cmd +++ b/ci/incremental/setup_conda_environment.cmd @@ -13,7 +13,7 @@ conda list @rem Clean up any left-over from a previous build conda remove --all -q -y -n %CONDA_ENV% @rem Scipy, CFFI, jinja2 and IPython are optional dependencies, but exercised in the test suite -conda env create -n %CONDA_ENV% --file=ci\azure-%CONDA_PY%.yaml +conda env create -n %CONDA_ENV% --file=ci\azure-windows-%CONDA_PY%.yaml call activate %CONDA_ENV% conda list From 3a8fadba0953b3e04faebe8c4b2de93c69c107be Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 19 Sep 2018 07:58:22 -0500 Subject: [PATCH 08/14] executable scripts --- ci/incremental/build.sh | 0 ci/incremental/install_miniconda.sh | 0 ci/incremental/setup_conda_environment.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/incremental/build.sh mode change 100644 => 100755 ci/incremental/install_miniconda.sh mode change 100644 => 100755 ci/incremental/setup_conda_environment.sh diff --git a/ci/incremental/build.sh b/ci/incremental/build.sh old mode 100644 new mode 100755 diff --git a/ci/incremental/install_miniconda.sh b/ci/incremental/install_miniconda.sh old mode 100644 new mode 100755 diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh old mode 100644 new mode 100755 From 8869e476a54cfd8043981c2bca823fb17da845ec Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 19 Sep 2018 08:25:06 -0500 Subject: [PATCH 09/14] Fixups * VC 9.0 for 2.7 * exit 0 in mac build --- ci/azure/windows-py27.yml | 9 +++++++++ ci/incremental/install_miniconda.sh | 6 +++--- ci/incremental/setup_conda_environment.sh | 3 ++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ci/azure/windows-py27.yml b/ci/azure/windows-py27.yml index 760310f88752e..e60844896b71c 100644 --- a/ci/azure/windows-py27.yml +++ b/ci/azure/windows-py27.yml @@ -20,6 +20,15 @@ jobs: updateConda: no packageSpecs: '' + # Need to install VC 9.0 only for Python 2.7 + # Once we understand how to do tasks conditional on build matrix variables + # we could merge this into azure-windows.yml + - powershell: | + $wc = New-Object net.webclient + $wc.Downloadfile("https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi", "VCForPython27.msi") + Start-Process "VCForPython27.msi" /qn -Wait + displayName: 'Install VC 9.0' + - script: | ci\\incremental\\setup_conda_environment.cmd displayName: 'Before Install' diff --git a/ci/incremental/install_miniconda.sh b/ci/incremental/install_miniconda.sh index 4aa73bac6729b..a47dfdb324b34 100755 --- a/ci/incremental/install_miniconda.sh +++ b/ci/incremental/install_miniconda.sh @@ -6,12 +6,12 @@ set -v -e unamestr=`uname` if [[ "$unamestr" == 'Linux' ]]; then if [[ "$BITS32" == "yes" ]]; then - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86.sh -O miniconda.sh + wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86.sh -O miniconda.sh else - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh fi elif [[ "$unamestr" == 'Darwin' ]]; then - wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh + wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh else echo Error fi diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index c5fa701f99ecd..15b39ab67d368 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -33,7 +33,8 @@ pip uninstall -y pandas echo echo "[no installed pandas]" conda list pandas -pip list --format columns |grep pandas +# Add the grep -v grep so that exit with 0 +pip list --format columns | grep pandas | {grep -v grep || true; } # # Install the compiler toolchain # if [[ $(uname) == Linux ]]; then From 91ab75cce04913f6a73b5bdc039b6fa7c9535e0e Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 19 Sep 2018 08:43:43 -0500 Subject: [PATCH 10/14] Just remove the check --- ci/incremental/setup_conda_environment.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index 15b39ab67d368..c716a39138644 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -33,8 +33,6 @@ pip uninstall -y pandas echo echo "[no installed pandas]" conda list pandas -# Add the grep -v grep so that exit with 0 -pip list --format columns | grep pandas | {grep -v grep || true; } # # Install the compiler toolchain # if [[ $(uname) == Linux ]]; then From 410391ee5997fc9b5932ffe26a8eda304a24aeb7 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 19 Sep 2018 09:01:41 -0500 Subject: [PATCH 11/14] Missing . --- ci/incremental/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/incremental/build.sh b/ci/incremental/build.sh index f5b5ecb5ab690..8f2301a3b7ef5 100755 --- a/ci/incremental/build.sh +++ b/ci/incremental/build.sh @@ -7,7 +7,7 @@ set -v -e echo "[building extensions]" python setup.py build_ext -q --inplace -python -m pip install -e +python -m pip install -e . echo echo "[show environment]" From 053fab230f20b7e1c2dba0294e54008f11255025 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 19 Sep 2018 09:13:03 -0500 Subject: [PATCH 12/14] Fixed path to scripts --- ci/azure/macos.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/azure/macos.yml b/ci/azure/macos.yml index 802b3722e3e27..25b66615dac7e 100644 --- a/ci/azure/macos.yml +++ b/ci/azure/macos.yml @@ -30,9 +30,9 @@ jobs: displayName: 'Build' - script: | export PATH=$HOME/miniconda3/bin:$PATH - ci/incremental/script_single.sh - ci/incremental/script_multi.sh - echo "Test done" + ci/script_single.sh + ci/script_multi.sh + echo "[Test done]" displayName: 'Test' - script: | export PATH=$HOME/miniconda3/bin:$PATH From a16965f1f0b81320772f73fd0f6275ff4e9441c6 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 19 Sep 2018 10:04:52 -0500 Subject: [PATCH 13/14] Remove appveyor --- ci/appveyor-27.yaml | 31 --------------- ci/appveyor-36.yaml | 28 -------------- ci/install.ps1 | 92 --------------------------------------------- 3 files changed, 151 deletions(-) delete mode 100644 ci/appveyor-27.yaml delete mode 100644 ci/appveyor-36.yaml delete mode 100644 ci/install.ps1 diff --git a/ci/appveyor-27.yaml b/ci/appveyor-27.yaml deleted file mode 100644 index bcd9ddee1715e..0000000000000 --- a/ci/appveyor-27.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: pandas -channels: - - defaults - - conda-forge -dependencies: - - beautifulsoup4 - - bottleneck - - dateutil - - gcsfs - - html5lib - - jinja2=2.8 - - lxml - - matplotlib - - numexpr - - numpy=1.12* - - openpyxl=2.5.5 - - pytables - - python=2.7.* - - pytz - - s3fs - - scipy - - sqlalchemy - - xlrd - - xlsxwriter - - xlwt - # universal - - cython>=0.28.2 - - pytest - - pytest-xdist - - moto - - hypothesis>=3.58.0 diff --git a/ci/appveyor-36.yaml b/ci/appveyor-36.yaml deleted file mode 100644 index 6230e9b6a1885..0000000000000 --- a/ci/appveyor-36.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: pandas -channels: - - defaults - - conda-forge -dependencies: - - blosc - - bottleneck - - fastparquet - - feather-format - - matplotlib - - numexpr - - numpy=1.14* - - openpyxl=2.5.5 - - pyarrow - - pytables - - python-dateutil - - python=3.6.* - - pytz - - scipy - - thrift=0.10* - - xlrd - - xlsxwriter - - xlwt - # universal - - cython>=0.28.2 - - pytest - - pytest-xdist - - hypothesis>=3.58.0 diff --git a/ci/install.ps1 b/ci/install.ps1 deleted file mode 100644 index 64ec7f81884cd..0000000000000 --- a/ci/install.ps1 +++ /dev/null @@ -1,92 +0,0 @@ -# Sample script to install Miniconda under Windows -# Authors: Olivier Grisel, Jonathan Helmus and Kyle Kastner, Robert McGibbon -# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ - -$MINICONDA_URL = "http://repo.continuum.io/miniconda/" - - -function DownloadMiniconda ($python_version, $platform_suffix) { - $webclient = New-Object System.Net.WebClient - $filename = "Miniconda3-latest-Windows-" + $platform_suffix + ".exe" - $url = $MINICONDA_URL + $filename - - $basedir = $pwd.Path + "\" - $filepath = $basedir + $filename - if (Test-Path $filename) { - Write-Host "Reusing" $filepath - return $filepath - } - - # Download and retry up to 3 times in case of network transient errors. - Write-Host "Downloading" $filename "from" $url - $retry_attempts = 2 - for($i=0; $i -lt $retry_attempts; $i++){ - try { - $webclient.DownloadFile($url, $filepath) - break - } - Catch [Exception]{ - Start-Sleep 1 - } - } - if (Test-Path $filepath) { - Write-Host "File saved at" $filepath - } else { - # Retry once to get the error message if any at the last try - $webclient.DownloadFile($url, $filepath) - } - return $filepath -} - - -function InstallMiniconda ($python_version, $architecture, $python_home) { - Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home - if (Test-Path $python_home) { - Write-Host $python_home "already exists, skipping." - return $false - } - if ($architecture -match "32") { - $platform_suffix = "x86" - } else { - $platform_suffix = "x86_64" - } - - $filepath = DownloadMiniconda $python_version $platform_suffix - Write-Host "Installing" $filepath "to" $python_home - $install_log = $python_home + ".log" - $args = "/S /D=$python_home" - Write-Host $filepath $args - Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru - if (Test-Path $python_home) { - Write-Host "Python $python_version ($architecture) installation complete" - } else { - Write-Host "Failed to install Python in $python_home" - Get-Content -Path $install_log - Exit 1 - } -} - - -function InstallCondaPackages ($python_home, $spec) { - $conda_path = $python_home + "\Scripts\conda.exe" - $args = "install --yes " + $spec - Write-Host ("conda " + $args) - Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru -} - -function UpdateConda ($python_home) { - $conda_path = $python_home + "\Scripts\conda.exe" - Write-Host "Updating conda..." - $args = "update --yes conda" - Write-Host $conda_path $args - Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru -} - - -function main () { - InstallMiniconda "3.5" $env:PYTHON_ARCH $env:CONDA_ROOT - UpdateConda $env:CONDA_ROOT - InstallCondaPackages $env:CONDA_ROOT "conda-build jinja2 anaconda-client" -} - -main From d09e2b7f6258201675c251c8fc7d9670e8b62b99 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 19 Sep 2018 10:05:38 -0500 Subject: [PATCH 14/14] Remove Travis OSX --- .travis.yml | 5 ----- ci/travis-35-osx.yaml | 28 ---------------------------- 2 files changed, 33 deletions(-) delete mode 100644 ci/travis-35-osx.yaml diff --git a/.travis.yml b/.travis.yml index 76f4715a4abb2..a180e83eeec21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,11 +30,6 @@ matrix: exclude: # Exclude the default Python 3.5 build - python: 3.5 - include: - - os: osx - language: generic - env: - - JOB="3.5, OSX" ENV_FILE="ci/travis-35-osx.yaml" TEST_ARGS="--skip-slow --skip-network" - dist: trusty env: diff --git a/ci/travis-35-osx.yaml b/ci/travis-35-osx.yaml deleted file mode 100644 index a36f748ded812..0000000000000 --- a/ci/travis-35-osx.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: pandas -channels: - - defaults -dependencies: - - beautifulsoup4 - - bottleneck - - cython>=0.28.2 - - html5lib - - jinja2 - - lxml - - matplotlib - - nomkl - - numexpr - - numpy=1.10.4 - - openpyxl=2.5.5 - - pytables - - python=3.5* - - pytz - - xarray - - xlrd - - xlsxwriter - - xlwt - # universal - - pytest - - pytest-xdist - - pip: - - python-dateutil==2.5.3 - - hypothesis>=3.58.0