From 4d277c4d6040413b8ae2e18fb0915308966b89b9 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 1 Oct 2019 10:58:01 -0500 Subject: [PATCH 01/41] poke at travis --- .travis.yml | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 79fecc41bec0d..85c344e76f9fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,27 +30,12 @@ matrix: - python: 3.5 include: - - dist: trusty + - dist: bionic + # 18.04 + python: 3.8-dev env: - - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)" + - JOB="3.8-dev" PATTERN="(not slow and not network)" - - dist: trusty - env: - - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" - - - dist: trusty - env: - - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true - - # In allow_failures - - dist: trusty - env: - - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" - - allow_failures: - - dist: trusty - env: - - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" before_install: - echo "before_install" @@ -74,7 +59,11 @@ before_install: install: - echo "install start" - ci/prep_cython_cache.sh - - ci/setup_env.sh + - git clone https://github.com/numpy/numpy + - pip install python-dateutil pytz cython + - cd numpy + - python setup.py build_ext --inplace + - python setup.py install - ci/submit_cython_cache.sh - echo "install done" From 56c49ca063bc1f52c2f220e38ab1f14287bf10aa Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 1 Oct 2019 12:08:59 -0500 Subject: [PATCH 02/41] troubleshoot --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 85c344e76f9fa..9d504de8dc55c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,12 +58,15 @@ before_install: install: - echo "install start" + - apt-get install build-essential - ci/prep_cython_cache.sh - git clone https://github.com/numpy/numpy + - pip install --no-deps -U pip wheel setuptools - pip install python-dateutil pytz cython - cd numpy - python setup.py build_ext --inplace - python setup.py install + - cd .. - ci/submit_cython_cache.sh - echo "install done" From 439571c7608b3a5dbb3d3caccf9387be3b4cc309 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 1 Oct 2019 15:31:22 -0500 Subject: [PATCH 03/41] add sudo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9d504de8dc55c..16858b0040fbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,7 +58,7 @@ before_install: install: - echo "install start" - - apt-get install build-essential + - sudo apt-get install build-essential - ci/prep_cython_cache.sh - git clone https://github.com/numpy/numpy - pip install --no-deps -U pip wheel setuptools From 66db6a61bfd8e0caaca7d5141ab568aaf3d1dbe7 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 1 Oct 2019 20:13:45 -0500 Subject: [PATCH 04/41] keep trying --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 16858b0040fbc..a527f5e44e8de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,6 +67,7 @@ install: - python setup.py build_ext --inplace - python setup.py install - cd .. + - ci/setup_env.sh - ci/submit_cython_cache.sh - echo "install done" From 80101f5f9087a0b66a389413e0a69ae1edb13194 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 1 Oct 2019 20:28:18 -0500 Subject: [PATCH 05/41] Add deps file --- .travis.yml | 2 +- ci/deps/travis-38.yaml | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 ci/deps/travis-38.yaml diff --git a/.travis.yml b/.travis.yml index a527f5e44e8de..209a0ba47a33f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ matrix: # 18.04 python: 3.8-dev env: - - JOB="3.8-dev" PATTERN="(not slow and not network)" + - JOB="3.8-dev" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network)" before_install: diff --git a/ci/deps/travis-38.yaml b/ci/deps/travis-38.yaml new file mode 100644 index 0000000000000..a7b7773e0023b --- /dev/null +++ b/ci/deps/travis-38.yaml @@ -0,0 +1,22 @@ +name: pandas-dev +channels: + - defaults + - conda-forge + - c3i_test +dependencies: + - python=3.8.* + - botocore>=1.11 + - cython>=0.29.13 + - python-dateutil + - nomkl + - pytz + # universal + - pytest>=5.0.0 + - pytest-xdist>=1.29.0 + - pytest-mock + - hypothesis>=3.58.0 + - s3fs<0.3 + - pip + - pyreadstat + - pip: + - moto From 2fce3b5670e49039e48ea98f6d557e7fa5e062ba Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 1 Oct 2019 20:39:54 -0500 Subject: [PATCH 06/41] again --- ci/deps/travis-38.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deps/travis-38.yaml b/ci/deps/travis-38.yaml index a7b7773e0023b..e3d85a4e388b7 100644 --- a/ci/deps/travis-38.yaml +++ b/ci/deps/travis-38.yaml @@ -4,7 +4,7 @@ channels: - conda-forge - c3i_test dependencies: - - python=3.8.* + #- python=3.8.* - botocore>=1.11 - cython>=0.29.13 - python-dateutil From 4eac0268b179bd32a878b64f3957ea8c4a232fad Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 1 Oct 2019 20:58:59 -0500 Subject: [PATCH 07/41] i said 38 --- .travis.yml | 2 +- ci/setup_env.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 209a0ba47a33f..a527f5e44e8de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ matrix: # 18.04 python: 3.8-dev env: - - JOB="3.8-dev" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network)" + - JOB="3.8-dev" PATTERN="(not slow and not network)" before_install: diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 382491a947488..c245bbe9ed6a8 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -91,8 +91,8 @@ conda list conda remove --all -q -y -n pandas-dev echo -echo "conda env create -q --file=${ENV_FILE}" -time conda env create -q --file="${ENV_FILE}" +echo "conda env create -q" +time conda env create -q if [[ "$BITS32" == "yes" ]]; then From ad56fe9ab265b315993b4bb43b49b3556c020a71 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 08:50:31 -0500 Subject: [PATCH 08/41] keep trying --- .travis.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index a527f5e44e8de..eec03fb3a6eaa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,26 +62,26 @@ install: - ci/prep_cython_cache.sh - git clone https://github.com/numpy/numpy - pip install --no-deps -U pip wheel setuptools - - pip install python-dateutil pytz cython + - pip install python-dateutil pytz cython pytest pytest-xdist hypothesis - cd numpy - python setup.py build_ext --inplace - python setup.py install - cd .. - - ci/setup_env.sh + #- ci/setup_env.sh - ci/submit_cython_cache.sh - echo "install done" -before_script: - # display server (for clipboard functionality) needs to be started here, - # does not work if done in install:setup_env.sh (GH-26103) - - export DISPLAY=":99.0" - - echo "sh -e /etc/init.d/xvfb start" - - sh -e /etc/init.d/xvfb start - - sleep 3 +#before_script: +# # display server (for clipboard functionality) needs to be started here, +# # does not work if done in install:setup_env.sh (GH-26103) +# - export DISPLAY=":99.0" +# - echo "sh -e /etc/init.d/xvfb start" +# - sh -e /etc/init.d/xvfb start +# - sleep 3 script: - echo "script start" - - source activate pandas-dev + #- source activate pandas-dev - ci/run_tests.sh after_script: From f4ad0201316284a540440fc5930f2ba41e46d554 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 09:25:29 -0500 Subject: [PATCH 09/41] do the build --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index eec03fb3a6eaa..4cbf08f21136a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,6 +67,7 @@ install: - python setup.py build_ext --inplace - python setup.py install - cd .. + - python setup.py build_ext -inplace #- ci/setup_env.sh - ci/submit_cython_cache.sh - echo "install done" From 4f985855ef2543dd5eb6cfea7a97aa11011512f6 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 09:44:54 -0500 Subject: [PATCH 10/41] maybe installing will help? --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4cbf08f21136a..e88b54b00723b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,6 +68,7 @@ install: - python setup.py install - cd .. - python setup.py build_ext -inplace + - python setup.py install #- ci/setup_env.sh - ci/submit_cython_cache.sh - echo "install done" From 209aecd37338da915d56da9d7b183793b31bd8f1 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 10:37:58 -0500 Subject: [PATCH 11/41] troubleshoot, disable azure --- .travis.yml | 7 +- azure-pipelines.yml | 180 -------------------------------------------- 2 files changed, 5 insertions(+), 182 deletions(-) delete mode 100644 azure-pipelines.yml diff --git a/.travis.yml b/.travis.yml index e88b54b00723b..07759325dbc1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,16 +59,19 @@ before_install: install: - echo "install start" - sudo apt-get install build-essential - - ci/prep_cython_cache.sh - - git clone https://github.com/numpy/numpy - pip install --no-deps -U pip wheel setuptools - pip install python-dateutil pytz cython pytest pytest-xdist hypothesis + - ci/prep_cython_cache.sh + - git clone https://github.com/numpy/numpy - cd numpy - python setup.py build_ext --inplace - python setup.py install - cd .. + - rm -rf numpy - python setup.py build_ext -inplace - python setup.py install + - python -c "import sys; print(sys.version_info)" + - python -c "import pandas as pd" #- ci/setup_env.sh - ci/submit_cython_cache.sh - echo "install done" diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 62c46b6970969..0000000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,180 +0,0 @@ -# Adapted from https://github.com/numba/numba/blob/master/azure-pipelines.yml -jobs: -# Mac and Linux use the same template -- template: ci/azure/posix.yml - parameters: - name: macOS - vmImage: xcode9-macos10.13 - -- template: ci/azure/posix.yml - parameters: - name: Linux - vmImage: ubuntu-16.04 - -- template: ci/azure/windows.yml - parameters: - name: Windows - vmImage: vs2017-win2016 - -- job: 'Checks' - pool: - vmImage: ubuntu-16.04 - timeoutInMinutes: 90 - steps: - - script: | - echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' - echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' - echo '##vso[task.setvariable variable=AZURE]true' - displayName: 'Setting environment variables' - - # Do not require a conda environment - - script: ci/code_checks.sh patterns - displayName: 'Looking for unwanted patterns' - condition: true - - - script: | - sudo apt-get install -y libc6-dev-i386 - ci/setup_env.sh - displayName: 'Setup environment and build pandas' - condition: true - - # Do not require pandas - - script: | - source activate pandas-dev - ci/code_checks.sh lint - displayName: 'Linting' - condition: true - - - script: | - source activate pandas-dev - ci/code_checks.sh dependencies - displayName: 'Dependencies consistency' - condition: true - - # Require pandas - - script: | - source activate pandas-dev - ci/code_checks.sh code - displayName: 'Checks on imported code' - condition: true - - - script: | - source activate pandas-dev - ci/code_checks.sh doctests - displayName: 'Running doctests' - condition: true - - - script: | - source activate pandas-dev - ci/code_checks.sh docstrings - displayName: 'Docstring validation' - condition: true - - - script: | - source activate pandas-dev - ci/code_checks.sh typing - displayName: 'Typing validation' - condition: true - - - script: | - source activate pandas-dev - pytest --capture=no --strict scripts - displayName: 'Testing docstring validation script' - condition: true - - - script: | - source activate pandas-dev - cd asv_bench - asv check -E existing - git remote add upstream https://github.com/pandas-dev/pandas.git - git fetch upstream - if git diff upstream/master --name-only | grep -q "^asv_bench/"; then - asv machine --yes - ASV_OUTPUT="$(asv dev)" - if [[ $(echo "$ASV_OUTPUT" | grep "failed") ]]; then - echo "##vso[task.logissue type=error]Benchmarks run with errors" - echo "$ASV_OUTPUT" - exit 1 - else - echo "Benchmarks run without errors" - fi - else - echo "Benchmarks did not run, no changes detected" - fi - displayName: 'Running benchmarks' - condition: true - -- job: 'Web_and_Docs' - pool: - vmImage: ubuntu-16.04 - timeoutInMinutes: 90 - steps: - - script: | - echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' - echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' - displayName: 'Setting environment variables' - - - script: | - sudo apt-get install -y libc6-dev-i386 - ci/setup_env.sh - displayName: 'Setup environment and build pandas' - - - script: | - source activate pandas-dev - python web/pandas_web.py web/pandas --target-path=web/build - displayName: 'Build website' - - - script: | - source activate pandas-dev - # Next we should simply have `doc/make.py --warnings-are-errors`, everything else is required because the ipython directive doesn't fail the build on errors (https://github.com/ipython/ipython/issues/11547) - doc/make.py --warnings-are-errors | tee sphinx.log ; SPHINX_RET=${PIPESTATUS[0]} - grep -B1 "^<<<-------------------------------------------------------------------------$" sphinx.log ; IPY_RET=$(( $? != 1 )) - exit $(( $SPHINX_RET + $IPY_RET )) - displayName: 'Build documentation' - - - script: | - mkdir -p to_deploy/docs - cp -r web/build/* to_deploy/ - cp -r doc/build/html/* to_deploy/docs/ - displayName: 'Merge website and docs' - - - script: | - cd to_deploy - git init - touch .nojekyll - echo "dev.pandas.io" > CNAME - printf "User-agent: *\nDisallow: /" > robots.txt - git add --all . - git config user.email "pandas-dev@python.org" - git config user.name "pandas-bot" - git commit -m "pandas web and documentation in master" - displayName: 'Create git repo for docs build' - condition : | - and(not(eq(variables['Build.Reason'], 'PullRequest')), - eq(variables['Build.SourceBranch'], 'refs/heads/master')) - - # For `InstallSSHKey@0` to work, next steps are required: - # 1. Generate a pair of private/public keys (i.e. `ssh-keygen -t rsa -b 4096 -C "your_email@example.com"`) - # 2. Go to "Library > Secure files" in the Azure Pipelines dashboard: https://dev.azure.com/pandas-dev/pandas/_library?itemType=SecureFiles - # 3. Click on "+ Secure file" - # 4. Upload the private key (the name of the file must match with the specified in "sshKeySecureFile" input below, "pandas_docs_key") - # 5. Click on file name after it is created, tick the box "Authorize for use in all pipelines" and save - # 6. The public key specified in "sshPublicKey" is the pair of the uploaded private key, and needs to be set as a deploy key of the repo where the docs will be pushed (with write access): https://github.com/pandas-dev/pandas-dev.github.io/settings/keys - - task: InstallSSHKey@0 - inputs: - hostName: 'github.com,192.30.252.128 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' - sshPublicKey: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDHmz3l/EdqrgNxEUKkwDUuUcLv91unig03pYFGO/DMIgCmPdMG96zAgfnESd837Rm0wSSqylwSzkRJt5MV/TpFlcVifDLDQmUhqCeO8Z6dLl/oe35UKmyYICVwcvQTAaHNnYRpKC5IUlTh0JEtw9fGlnp1Ta7U1ENBLbKdpywczElhZu+hOQ892zqOj3CwA+U2329/d6cd7YnqIKoFN9DWT3kS5K6JE4IoBfQEVekIOs23bKjNLvPoOmi6CroAhu/K8j+NCWQjge5eJf2x/yTnIIP1PlEcXoHIr8io517posIx3TBup+CN8bNS1PpDW3jyD3ttl1uoBudjOQrobNnJeR6Rn67DRkG6IhSwr3BWj8alwUG5mTdZzwV5Pa9KZFdIiqX7NoDGg+itsR39QCn0thK8lGRNSR8KrWC1PSjecwelKBO7uQ7rnk/rkrZdBWR4oEA8YgNH8tirUw5WfOr5a0AIaJicKxGKNdMxZt+zmC+bS7F4YCOGIm9KHa43RrKhoGRhRf9fHHHKUPwFGqtWG4ykcUgoamDOURJyepesBAO3FiRE9rLU6ILbB3yEqqoekborHmAJD5vf7PWItW3Q/YQKuk3kkqRcKnexPyzyyq5lUgTi8CxxZdaASIOu294wjBhhdyHlXEkVTNJ9JKkj/obF+XiIIp0cBDsOXY9hDQ== pandas-dev@python.org' - sshKeySecureFile: 'pandas_docs_key' - displayName: 'Install GitHub ssh deployment key' - condition : | - and(not(eq(variables['Build.Reason'], 'PullRequest')), - eq(variables['Build.SourceBranch'], 'refs/heads/master')) - - - script: | - cd to_deploy - git remote add origin git@github.com:pandas-dev/pandas-dev.github.io.git - git push -f origin master - displayName: 'Publish web and docs to GitHub pages' - condition : | - and(not(eq(variables['Build.Reason'], 'PullRequest')), - eq(variables['Build.SourceBranch'], 'refs/heads/master')) From 7d39c4dcc3ed4e309336be6ca2ff0bc3399244d3 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 12:01:09 -0500 Subject: [PATCH 12/41] install using pip --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 07759325dbc1e..1365e6db50b14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,7 +69,8 @@ install: - cd .. - rm -rf numpy - python setup.py build_ext -inplace - - python setup.py install + - python -m pip install --no-build-isolation -e . + #- python setup.py install - python -c "import sys; print(sys.version_info)" - python -c "import pandas as pd" #- ci/setup_env.sh From 6c7e0d1aaf75b07a490bca2cf14ce6963014b1c0 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 3 Oct 2019 11:51:51 -0500 Subject: [PATCH 13/41] troubleshoot --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1365e6db50b14..f16133d9d3cdc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,6 +73,7 @@ install: #- python setup.py install - python -c "import sys; print(sys.version_info)" - python -c "import pandas as pd" + - python -c "import hypothesis" #- ci/setup_env.sh - ci/submit_cython_cache.sh - echo "install done" @@ -88,7 +89,8 @@ install: script: - echo "script start" #- source activate pandas-dev - - ci/run_tests.sh + #- ci/run_tests.sh + - pytest pandas/tests --skip-slow after_script: - echo "after_script start" From 2e28dca71d1f8042e1614d411eceffed1612192a Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 3 Oct 2019 12:45:39 -0500 Subject: [PATCH 14/41] working with just pytest, now try with run_tests --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f16133d9d3cdc..fc4844fb37763 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,8 +89,8 @@ install: script: - echo "script start" #- source activate pandas-dev - #- ci/run_tests.sh - - pytest pandas/tests --skip-slow + - ci/run_tests.sh + #- pytest pandas/tests --skip-slow after_script: - echo "after_script start" From 452e251e0fc856f0f6c3256d726d2916d7aaad16 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 3 Oct 2019 15:08:47 -0500 Subject: [PATCH 15/41] restore, try to do stuff conditionally --- .travis.yml | 72 ++++++++++++++++++++++++++++++--------------------- ci/build38.sh | 30 +++++++++++++++++++++ 2 files changed, 72 insertions(+), 30 deletions(-) create mode 100644 ci/build38.sh diff --git a/.travis.yml b/.travis.yml index fc4844fb37763..20b7332bab1f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,27 @@ matrix: env: - JOB="3.8-dev" PATTERN="(not slow and not network)" + - dist: trusty + env: + - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)" + + - dist: trusty + env: + - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" + + - dist: trusty + env: + - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true + + # In allow_failures + - dist: trusty + env: + - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" + + allow_failures: + - dist: trusty + env: + - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" before_install: - echo "before_install" @@ -56,41 +77,32 @@ before_install: # This overrides travis and tells it to look nowhere. - export BOTO_CONFIG=/dev/null + install: - - echo "install start" - - sudo apt-get install build-essential - - pip install --no-deps -U pip wheel setuptools - - pip install python-dateutil pytz cython pytest pytest-xdist hypothesis - - ci/prep_cython_cache.sh - - git clone https://github.com/numpy/numpy - - cd numpy - - python setup.py build_ext --inplace - - python setup.py install - - cd .. - - rm -rf numpy - - python setup.py build_ext -inplace - - python -m pip install --no-build-isolation -e . - #- python setup.py install - - python -c "import sys; print(sys.version_info)" - - python -c "import pandas as pd" - - python -c "import hypothesis" - #- ci/setup_env.sh - - ci/submit_cython_cache.sh - - echo "install done" - -#before_script: -# # display server (for clipboard functionality) needs to be started here, -# # does not work if done in install:setup_env.sh (GH-26103) -# - export DISPLAY=":99.0" -# - echo "sh -e /etc/init.d/xvfb start" -# - sh -e /etc/init.d/xvfb start -# - sleep 3 + if: env(JOB) = "3.8-dev" + - ci/build38.sh + if: env(JOB) != "3.8-dev" + - echo "install start" + - ci/prep_cython_cache.sh + - ci/setup_env.sh + - ci/submit_cython_cache.sh + - echo "install done" + + +before_script: + # display server (for clipboard functionality) needs to be started here, + # does not work if done in install:setup_env.sh (GH-26103) + if: env(JOB) != "3.8-dev" + - export DISPLAY=":99.0" + - echo "sh -e /etc/init.d/xvfb start" + - sh -e /etc/init.d/xvfb start + - sleep 3 script: - echo "script start" - #- source activate pandas-dev + if: env(JOB) != "3.8-dev" + - source activate pandas-dev - ci/run_tests.sh - #- pytest pandas/tests --skip-slow after_script: - echo "after_script start" diff --git a/ci/build38.sh b/ci/build38.sh new file mode 100644 index 0000000000000..b21b356c4df63 --- /dev/null +++ b/ci/build38.sh @@ -0,0 +1,30 @@ +#!/bin/bash -e +# Special build for python3.8 until numpy puts its own wheels up + +echo "python3.8 install start" + +sudo apt-get install build-essential +pip install --no-deps -U pip wheel setuptools +pip install python-dateutil pytz cython pytest pytest-xdist hypothesis + +ci/prep_cython_cache.sh + +git clone https://github.com/numpy/numpy +cd numpy +python setup.py build_ext --inplace +python setup.py install +cd .. +rm -rf numpy + +python setup.py build_ext -inplace +python -m pip install --no-build-isolation -e . + +python -c "import sys; print(sys.version_info)" +python -c "import pandas as pd" +python -c "import hypothesis" + +# TODO: Is there anything else in setup_env that we really want to do? +# ci/setup_env.sh + +ci/submit_cython_cache.sh +echo "python3.8 install done" From 3c8297d8c2f16df70d85d29b004ecb00224598e4 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 3 Oct 2019 15:13:46 -0500 Subject: [PATCH 16/41] try to guess how conditional syntax works on travis --- .travis.yml | 30 +++++++++++++++--------------- ci/build38.sh | 7 ------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 20b7332bab1f2..d30ceadaa41d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,29 +79,29 @@ before_install: install: - if: env(JOB) = "3.8-dev" - - ci/build38.sh - if: env(JOB) != "3.8-dev" - - echo "install start" - - ci/prep_cython_cache.sh - - ci/setup_env.sh - - ci/submit_cython_cache.sh - - echo "install done" + - echo "install start" + - ci/prep_cython_cache.sh + - ci/build38.sh + if: env(JOB) = "3.8-dev" + - ci/setup_env.sh + if: env(JOB) != "3.8-dev" + - ci/submit_cython_cache.sh + - echo "install done" before_script: # display server (for clipboard functionality) needs to be started here, # does not work if done in install:setup_env.sh (GH-26103) - if: env(JOB) != "3.8-dev" - - export DISPLAY=":99.0" - - echo "sh -e /etc/init.d/xvfb start" - - sh -e /etc/init.d/xvfb start - - sleep 3 + - if: env(JOB) != "3.8-dev" + - export DISPLAY=":99.0" + - echo "sh -e /etc/init.d/xvfb start" + - sh -e /etc/init.d/xvfb start + - sleep 3 script: - echo "script start" - if: env(JOB) != "3.8-dev" - - source activate pandas-dev + - source activate pandas-dev + if: env(JOB) != "3.8-dev" - ci/run_tests.sh after_script: diff --git a/ci/build38.sh b/ci/build38.sh index b21b356c4df63..d72bce22a3cce 100644 --- a/ci/build38.sh +++ b/ci/build38.sh @@ -1,14 +1,10 @@ #!/bin/bash -e # Special build for python3.8 until numpy puts its own wheels up -echo "python3.8 install start" - sudo apt-get install build-essential pip install --no-deps -U pip wheel setuptools pip install python-dateutil pytz cython pytest pytest-xdist hypothesis -ci/prep_cython_cache.sh - git clone https://github.com/numpy/numpy cd numpy python setup.py build_ext --inplace @@ -25,6 +21,3 @@ python -c "import hypothesis" # TODO: Is there anything else in setup_env that we really want to do? # ci/setup_env.sh - -ci/submit_cython_cache.sh -echo "python3.8 install done" From 61f69a0b356fdeed847219b0e1d6f8db8058eb29 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 3 Oct 2019 15:14:57 -0500 Subject: [PATCH 17/41] revert --- azure-pipelines.yml | 180 +++++++++++++++++++++++++++++++++++++++++ ci/deps/travis-38.yaml | 22 ----- ci/setup_env.sh | 4 +- 3 files changed, 182 insertions(+), 24 deletions(-) create mode 100644 azure-pipelines.yml delete mode 100644 ci/deps/travis-38.yaml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000000000..62c46b6970969 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,180 @@ +# Adapted from https://github.com/numba/numba/blob/master/azure-pipelines.yml +jobs: +# Mac and Linux use the same template +- template: ci/azure/posix.yml + parameters: + name: macOS + vmImage: xcode9-macos10.13 + +- template: ci/azure/posix.yml + parameters: + name: Linux + vmImage: ubuntu-16.04 + +- template: ci/azure/windows.yml + parameters: + name: Windows + vmImage: vs2017-win2016 + +- job: 'Checks' + pool: + vmImage: ubuntu-16.04 + timeoutInMinutes: 90 + steps: + - script: | + echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' + echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' + echo '##vso[task.setvariable variable=AZURE]true' + displayName: 'Setting environment variables' + + # Do not require a conda environment + - script: ci/code_checks.sh patterns + displayName: 'Looking for unwanted patterns' + condition: true + + - script: | + sudo apt-get install -y libc6-dev-i386 + ci/setup_env.sh + displayName: 'Setup environment and build pandas' + condition: true + + # Do not require pandas + - script: | + source activate pandas-dev + ci/code_checks.sh lint + displayName: 'Linting' + condition: true + + - script: | + source activate pandas-dev + ci/code_checks.sh dependencies + displayName: 'Dependencies consistency' + condition: true + + # Require pandas + - script: | + source activate pandas-dev + ci/code_checks.sh code + displayName: 'Checks on imported code' + condition: true + + - script: | + source activate pandas-dev + ci/code_checks.sh doctests + displayName: 'Running doctests' + condition: true + + - script: | + source activate pandas-dev + ci/code_checks.sh docstrings + displayName: 'Docstring validation' + condition: true + + - script: | + source activate pandas-dev + ci/code_checks.sh typing + displayName: 'Typing validation' + condition: true + + - script: | + source activate pandas-dev + pytest --capture=no --strict scripts + displayName: 'Testing docstring validation script' + condition: true + + - script: | + source activate pandas-dev + cd asv_bench + asv check -E existing + git remote add upstream https://github.com/pandas-dev/pandas.git + git fetch upstream + if git diff upstream/master --name-only | grep -q "^asv_bench/"; then + asv machine --yes + ASV_OUTPUT="$(asv dev)" + if [[ $(echo "$ASV_OUTPUT" | grep "failed") ]]; then + echo "##vso[task.logissue type=error]Benchmarks run with errors" + echo "$ASV_OUTPUT" + exit 1 + else + echo "Benchmarks run without errors" + fi + else + echo "Benchmarks did not run, no changes detected" + fi + displayName: 'Running benchmarks' + condition: true + +- job: 'Web_and_Docs' + pool: + vmImage: ubuntu-16.04 + timeoutInMinutes: 90 + steps: + - script: | + echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' + echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' + displayName: 'Setting environment variables' + + - script: | + sudo apt-get install -y libc6-dev-i386 + ci/setup_env.sh + displayName: 'Setup environment and build pandas' + + - script: | + source activate pandas-dev + python web/pandas_web.py web/pandas --target-path=web/build + displayName: 'Build website' + + - script: | + source activate pandas-dev + # Next we should simply have `doc/make.py --warnings-are-errors`, everything else is required because the ipython directive doesn't fail the build on errors (https://github.com/ipython/ipython/issues/11547) + doc/make.py --warnings-are-errors | tee sphinx.log ; SPHINX_RET=${PIPESTATUS[0]} + grep -B1 "^<<<-------------------------------------------------------------------------$" sphinx.log ; IPY_RET=$(( $? != 1 )) + exit $(( $SPHINX_RET + $IPY_RET )) + displayName: 'Build documentation' + + - script: | + mkdir -p to_deploy/docs + cp -r web/build/* to_deploy/ + cp -r doc/build/html/* to_deploy/docs/ + displayName: 'Merge website and docs' + + - script: | + cd to_deploy + git init + touch .nojekyll + echo "dev.pandas.io" > CNAME + printf "User-agent: *\nDisallow: /" > robots.txt + git add --all . + git config user.email "pandas-dev@python.org" + git config user.name "pandas-bot" + git commit -m "pandas web and documentation in master" + displayName: 'Create git repo for docs build' + condition : | + and(not(eq(variables['Build.Reason'], 'PullRequest')), + eq(variables['Build.SourceBranch'], 'refs/heads/master')) + + # For `InstallSSHKey@0` to work, next steps are required: + # 1. Generate a pair of private/public keys (i.e. `ssh-keygen -t rsa -b 4096 -C "your_email@example.com"`) + # 2. Go to "Library > Secure files" in the Azure Pipelines dashboard: https://dev.azure.com/pandas-dev/pandas/_library?itemType=SecureFiles + # 3. Click on "+ Secure file" + # 4. Upload the private key (the name of the file must match with the specified in "sshKeySecureFile" input below, "pandas_docs_key") + # 5. Click on file name after it is created, tick the box "Authorize for use in all pipelines" and save + # 6. The public key specified in "sshPublicKey" is the pair of the uploaded private key, and needs to be set as a deploy key of the repo where the docs will be pushed (with write access): https://github.com/pandas-dev/pandas-dev.github.io/settings/keys + - task: InstallSSHKey@0 + inputs: + hostName: 'github.com,192.30.252.128 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' + sshPublicKey: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDHmz3l/EdqrgNxEUKkwDUuUcLv91unig03pYFGO/DMIgCmPdMG96zAgfnESd837Rm0wSSqylwSzkRJt5MV/TpFlcVifDLDQmUhqCeO8Z6dLl/oe35UKmyYICVwcvQTAaHNnYRpKC5IUlTh0JEtw9fGlnp1Ta7U1ENBLbKdpywczElhZu+hOQ892zqOj3CwA+U2329/d6cd7YnqIKoFN9DWT3kS5K6JE4IoBfQEVekIOs23bKjNLvPoOmi6CroAhu/K8j+NCWQjge5eJf2x/yTnIIP1PlEcXoHIr8io517posIx3TBup+CN8bNS1PpDW3jyD3ttl1uoBudjOQrobNnJeR6Rn67DRkG6IhSwr3BWj8alwUG5mTdZzwV5Pa9KZFdIiqX7NoDGg+itsR39QCn0thK8lGRNSR8KrWC1PSjecwelKBO7uQ7rnk/rkrZdBWR4oEA8YgNH8tirUw5WfOr5a0AIaJicKxGKNdMxZt+zmC+bS7F4YCOGIm9KHa43RrKhoGRhRf9fHHHKUPwFGqtWG4ykcUgoamDOURJyepesBAO3FiRE9rLU6ILbB3yEqqoekborHmAJD5vf7PWItW3Q/YQKuk3kkqRcKnexPyzyyq5lUgTi8CxxZdaASIOu294wjBhhdyHlXEkVTNJ9JKkj/obF+XiIIp0cBDsOXY9hDQ== pandas-dev@python.org' + sshKeySecureFile: 'pandas_docs_key' + displayName: 'Install GitHub ssh deployment key' + condition : | + and(not(eq(variables['Build.Reason'], 'PullRequest')), + eq(variables['Build.SourceBranch'], 'refs/heads/master')) + + - script: | + cd to_deploy + git remote add origin git@github.com:pandas-dev/pandas-dev.github.io.git + git push -f origin master + displayName: 'Publish web and docs to GitHub pages' + condition : | + and(not(eq(variables['Build.Reason'], 'PullRequest')), + eq(variables['Build.SourceBranch'], 'refs/heads/master')) diff --git a/ci/deps/travis-38.yaml b/ci/deps/travis-38.yaml deleted file mode 100644 index e3d85a4e388b7..0000000000000 --- a/ci/deps/travis-38.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: pandas-dev -channels: - - defaults - - conda-forge - - c3i_test -dependencies: - #- python=3.8.* - - botocore>=1.11 - - cython>=0.29.13 - - python-dateutil - - nomkl - - pytz - # universal - - pytest>=5.0.0 - - pytest-xdist>=1.29.0 - - pytest-mock - - hypothesis>=3.58.0 - - s3fs<0.3 - - pip - - pyreadstat - - pip: - - moto diff --git a/ci/setup_env.sh b/ci/setup_env.sh index c245bbe9ed6a8..382491a947488 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -91,8 +91,8 @@ conda list conda remove --all -q -y -n pandas-dev echo -echo "conda env create -q" -time conda env create -q +echo "conda env create -q --file=${ENV_FILE}" +time conda env create -q --file="${ENV_FILE}" if [[ "$BITS32" == "yes" ]]; then From 82dbbb1b219e2ef593cd717f4265261eb26d1fc0 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 9 Oct 2019 13:25:06 -0700 Subject: [PATCH 18/41] try it without conditionals in yaml --- .travis.yml | 7 +------ ci/build38.sh | 2 +- ci/setup_env.sh | 4 ++++ 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index d30ceadaa41d1..fec61bacbef08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,10 +81,7 @@ before_install: install: - echo "install start" - ci/prep_cython_cache.sh - - ci/build38.sh - if: env(JOB) = "3.8-dev" - ci/setup_env.sh - if: env(JOB) != "3.8-dev" - ci/submit_cython_cache.sh - echo "install done" @@ -92,7 +89,6 @@ install: before_script: # display server (for clipboard functionality) needs to be started here, # does not work if done in install:setup_env.sh (GH-26103) - - if: env(JOB) != "3.8-dev" - export DISPLAY=":99.0" - echo "sh -e /etc/init.d/xvfb start" - sh -e /etc/init.d/xvfb start @@ -100,8 +96,7 @@ before_script: script: - echo "script start" - - source activate pandas-dev - if: env(JOB) != "3.8-dev" + - if [ $JOB != "3.8-dev"]; then source activate pandas-dev; fi - ci/run_tests.sh after_script: diff --git a/ci/build38.sh b/ci/build38.sh index d72bce22a3cce..302839ebdba04 100644 --- a/ci/build38.sh +++ b/ci/build38.sh @@ -1,7 +1,7 @@ #!/bin/bash -e # Special build for python3.8 until numpy puts its own wheels up -sudo apt-get install build-essential +sudo apt-get install build-essential xvfb pip install --no-deps -U pip wheel setuptools pip install python-dateutil pytz cython pytest pytest-xdist hypothesis diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 382491a947488..be8c3645691fe 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -1,5 +1,9 @@ #!/bin/bash -e +if [ "$JOB" == "3.8-dev" ]; then + /bin/bash ci/build38.sh + exit 0 +fi # edit the locale file if needed if [ -n "$LOCALE_OVERRIDE" ]; then From a33105f6c107d050869a2daad07adb698ad771ef Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 9 Oct 2019 14:35:52 -0700 Subject: [PATCH 19/41] troubleshoot --- .travis.yml | 3 ++- ci/build38.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index fec61bacbef08..aa1f616add79f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -96,7 +96,8 @@ before_script: script: - echo "script start" - - if [ $JOB != "3.8-dev"]; then source activate pandas-dev; fi + - echo "$JOB" + - if [ "$JOB" != "3.8-dev" ]; then source activate pandas-dev; fi - ci/run_tests.sh after_script: diff --git a/ci/build38.sh b/ci/build38.sh index 302839ebdba04..8f1c7dff1d24f 100644 --- a/ci/build38.sh +++ b/ci/build38.sh @@ -1,7 +1,7 @@ #!/bin/bash -e # Special build for python3.8 until numpy puts its own wheels up -sudo apt-get install build-essential xvfb +sudo apt-get install build-essential gcc xvfb pip install --no-deps -U pip wheel setuptools pip install python-dateutil pytz cython pytest pytest-xdist hypothesis From 056129440b361999a8a6ae063bb23bcb8721be28 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 9 Oct 2019 15:43:15 -0700 Subject: [PATCH 20/41] disable xvfb on 3.8-dev --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index aa1f616add79f..b9fa06304d387 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,7 +91,7 @@ before_script: # does not work if done in install:setup_env.sh (GH-26103) - export DISPLAY=":99.0" - echo "sh -e /etc/init.d/xvfb start" - - sh -e /etc/init.d/xvfb start + - if [ "$JOB" != "3.8-dev" ]; then sh -e /etc/init.d/xvfb start; fi - sleep 3 script: From 8cdaaf7786687bbb5d28f62c4e035a4cbcb47f49 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 9 Oct 2019 20:49:05 -0700 Subject: [PATCH 21/41] use pip for numpy --- ci/build38.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ci/build38.sh b/ci/build38.sh index 8f1c7dff1d24f..1b35fea4c88d8 100644 --- a/ci/build38.sh +++ b/ci/build38.sh @@ -5,12 +5,7 @@ sudo apt-get install build-essential gcc xvfb pip install --no-deps -U pip wheel setuptools pip install python-dateutil pytz cython pytest pytest-xdist hypothesis -git clone https://github.com/numpy/numpy -cd numpy -python setup.py build_ext --inplace -python setup.py install -cd .. -rm -rf numpy +pip install --pre -f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com/ numpy python setup.py build_ext -inplace python -m pip install --no-build-isolation -e . From fcbf9215c8ae6d50f1689e3bea15515239cacf95 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 1 Oct 2019 10:58:01 -0500 Subject: [PATCH 22/41] poke at travis --- .travis.yml | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 79fecc41bec0d..85c344e76f9fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,27 +30,12 @@ matrix: - python: 3.5 include: - - dist: trusty + - dist: bionic + # 18.04 + python: 3.8-dev env: - - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)" + - JOB="3.8-dev" PATTERN="(not slow and not network)" - - dist: trusty - env: - - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" - - - dist: trusty - env: - - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true - - # In allow_failures - - dist: trusty - env: - - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" - - allow_failures: - - dist: trusty - env: - - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" before_install: - echo "before_install" @@ -74,7 +59,11 @@ before_install: install: - echo "install start" - ci/prep_cython_cache.sh - - ci/setup_env.sh + - git clone https://github.com/numpy/numpy + - pip install python-dateutil pytz cython + - cd numpy + - python setup.py build_ext --inplace + - python setup.py install - ci/submit_cython_cache.sh - echo "install done" From 62a1d00646e86e27ca2e6e6c3ea8fa6f246c4003 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 1 Oct 2019 12:08:59 -0500 Subject: [PATCH 23/41] troubleshoot --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 85c344e76f9fa..9d504de8dc55c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,12 +58,15 @@ before_install: install: - echo "install start" + - apt-get install build-essential - ci/prep_cython_cache.sh - git clone https://github.com/numpy/numpy + - pip install --no-deps -U pip wheel setuptools - pip install python-dateutil pytz cython - cd numpy - python setup.py build_ext --inplace - python setup.py install + - cd .. - ci/submit_cython_cache.sh - echo "install done" From 426ba4a30046665be9558f2535889c065e11caad Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 1 Oct 2019 15:31:22 -0500 Subject: [PATCH 24/41] add sudo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9d504de8dc55c..16858b0040fbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,7 +58,7 @@ before_install: install: - echo "install start" - - apt-get install build-essential + - sudo apt-get install build-essential - ci/prep_cython_cache.sh - git clone https://github.com/numpy/numpy - pip install --no-deps -U pip wheel setuptools From d93e5a62f9e88b514da1d0e88f8b82611728ff11 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 1 Oct 2019 20:13:45 -0500 Subject: [PATCH 25/41] keep trying --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 16858b0040fbc..a527f5e44e8de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,6 +67,7 @@ install: - python setup.py build_ext --inplace - python setup.py install - cd .. + - ci/setup_env.sh - ci/submit_cython_cache.sh - echo "install done" From 3f840251f7bfd48d0e8eb182c24caf6b863c823c Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 1 Oct 2019 20:28:18 -0500 Subject: [PATCH 26/41] Add deps file --- .travis.yml | 2 +- ci/deps/travis-38.yaml | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 ci/deps/travis-38.yaml diff --git a/.travis.yml b/.travis.yml index a527f5e44e8de..209a0ba47a33f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ matrix: # 18.04 python: 3.8-dev env: - - JOB="3.8-dev" PATTERN="(not slow and not network)" + - JOB="3.8-dev" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network)" before_install: diff --git a/ci/deps/travis-38.yaml b/ci/deps/travis-38.yaml new file mode 100644 index 0000000000000..a7b7773e0023b --- /dev/null +++ b/ci/deps/travis-38.yaml @@ -0,0 +1,22 @@ +name: pandas-dev +channels: + - defaults + - conda-forge + - c3i_test +dependencies: + - python=3.8.* + - botocore>=1.11 + - cython>=0.29.13 + - python-dateutil + - nomkl + - pytz + # universal + - pytest>=5.0.0 + - pytest-xdist>=1.29.0 + - pytest-mock + - hypothesis>=3.58.0 + - s3fs<0.3 + - pip + - pyreadstat + - pip: + - moto From 64da0dba9141e166b8af4a7f620d85428378ea19 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 1 Oct 2019 20:39:54 -0500 Subject: [PATCH 27/41] again --- ci/deps/travis-38.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deps/travis-38.yaml b/ci/deps/travis-38.yaml index a7b7773e0023b..e3d85a4e388b7 100644 --- a/ci/deps/travis-38.yaml +++ b/ci/deps/travis-38.yaml @@ -4,7 +4,7 @@ channels: - conda-forge - c3i_test dependencies: - - python=3.8.* + #- python=3.8.* - botocore>=1.11 - cython>=0.29.13 - python-dateutil From e6fdeff0fb44a4d87f6dd3f07930667c9880ef47 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 1 Oct 2019 20:58:59 -0500 Subject: [PATCH 28/41] i said 38 --- .travis.yml | 2 +- ci/setup_env.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 209a0ba47a33f..a527f5e44e8de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ matrix: # 18.04 python: 3.8-dev env: - - JOB="3.8-dev" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network)" + - JOB="3.8-dev" PATTERN="(not slow and not network)" before_install: diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 382491a947488..c245bbe9ed6a8 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -91,8 +91,8 @@ conda list conda remove --all -q -y -n pandas-dev echo -echo "conda env create -q --file=${ENV_FILE}" -time conda env create -q --file="${ENV_FILE}" +echo "conda env create -q" +time conda env create -q if [[ "$BITS32" == "yes" ]]; then From 02db7c3892d79285b8163c166833b530746a9b0f Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 08:50:31 -0500 Subject: [PATCH 29/41] keep trying --- .travis.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index a527f5e44e8de..eec03fb3a6eaa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,26 +62,26 @@ install: - ci/prep_cython_cache.sh - git clone https://github.com/numpy/numpy - pip install --no-deps -U pip wheel setuptools - - pip install python-dateutil pytz cython + - pip install python-dateutil pytz cython pytest pytest-xdist hypothesis - cd numpy - python setup.py build_ext --inplace - python setup.py install - cd .. - - ci/setup_env.sh + #- ci/setup_env.sh - ci/submit_cython_cache.sh - echo "install done" -before_script: - # display server (for clipboard functionality) needs to be started here, - # does not work if done in install:setup_env.sh (GH-26103) - - export DISPLAY=":99.0" - - echo "sh -e /etc/init.d/xvfb start" - - sh -e /etc/init.d/xvfb start - - sleep 3 +#before_script: +# # display server (for clipboard functionality) needs to be started here, +# # does not work if done in install:setup_env.sh (GH-26103) +# - export DISPLAY=":99.0" +# - echo "sh -e /etc/init.d/xvfb start" +# - sh -e /etc/init.d/xvfb start +# - sleep 3 script: - echo "script start" - - source activate pandas-dev + #- source activate pandas-dev - ci/run_tests.sh after_script: From fbf1421cee3949201af2c4d6015489a0a5f191c3 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 09:25:29 -0500 Subject: [PATCH 30/41] do the build --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index eec03fb3a6eaa..4cbf08f21136a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,6 +67,7 @@ install: - python setup.py build_ext --inplace - python setup.py install - cd .. + - python setup.py build_ext -inplace #- ci/setup_env.sh - ci/submit_cython_cache.sh - echo "install done" From 48f98e9ec92af6c3551ed327eae8feccde127e0c Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 09:44:54 -0500 Subject: [PATCH 31/41] maybe installing will help? --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4cbf08f21136a..e88b54b00723b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,6 +68,7 @@ install: - python setup.py install - cd .. - python setup.py build_ext -inplace + - python setup.py install #- ci/setup_env.sh - ci/submit_cython_cache.sh - echo "install done" From fd49724d3b416e0a86b3d347975ca068a1c69472 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 10:37:58 -0500 Subject: [PATCH 32/41] troubleshoot, disable azure --- .travis.yml | 7 +- azure-pipelines.yml | 180 -------------------------------------------- 2 files changed, 5 insertions(+), 182 deletions(-) delete mode 100644 azure-pipelines.yml diff --git a/.travis.yml b/.travis.yml index e88b54b00723b..07759325dbc1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,16 +59,19 @@ before_install: install: - echo "install start" - sudo apt-get install build-essential - - ci/prep_cython_cache.sh - - git clone https://github.com/numpy/numpy - pip install --no-deps -U pip wheel setuptools - pip install python-dateutil pytz cython pytest pytest-xdist hypothesis + - ci/prep_cython_cache.sh + - git clone https://github.com/numpy/numpy - cd numpy - python setup.py build_ext --inplace - python setup.py install - cd .. + - rm -rf numpy - python setup.py build_ext -inplace - python setup.py install + - python -c "import sys; print(sys.version_info)" + - python -c "import pandas as pd" #- ci/setup_env.sh - ci/submit_cython_cache.sh - echo "install done" diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 62c46b6970969..0000000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,180 +0,0 @@ -# Adapted from https://github.com/numba/numba/blob/master/azure-pipelines.yml -jobs: -# Mac and Linux use the same template -- template: ci/azure/posix.yml - parameters: - name: macOS - vmImage: xcode9-macos10.13 - -- template: ci/azure/posix.yml - parameters: - name: Linux - vmImage: ubuntu-16.04 - -- template: ci/azure/windows.yml - parameters: - name: Windows - vmImage: vs2017-win2016 - -- job: 'Checks' - pool: - vmImage: ubuntu-16.04 - timeoutInMinutes: 90 - steps: - - script: | - echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' - echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' - echo '##vso[task.setvariable variable=AZURE]true' - displayName: 'Setting environment variables' - - # Do not require a conda environment - - script: ci/code_checks.sh patterns - displayName: 'Looking for unwanted patterns' - condition: true - - - script: | - sudo apt-get install -y libc6-dev-i386 - ci/setup_env.sh - displayName: 'Setup environment and build pandas' - condition: true - - # Do not require pandas - - script: | - source activate pandas-dev - ci/code_checks.sh lint - displayName: 'Linting' - condition: true - - - script: | - source activate pandas-dev - ci/code_checks.sh dependencies - displayName: 'Dependencies consistency' - condition: true - - # Require pandas - - script: | - source activate pandas-dev - ci/code_checks.sh code - displayName: 'Checks on imported code' - condition: true - - - script: | - source activate pandas-dev - ci/code_checks.sh doctests - displayName: 'Running doctests' - condition: true - - - script: | - source activate pandas-dev - ci/code_checks.sh docstrings - displayName: 'Docstring validation' - condition: true - - - script: | - source activate pandas-dev - ci/code_checks.sh typing - displayName: 'Typing validation' - condition: true - - - script: | - source activate pandas-dev - pytest --capture=no --strict scripts - displayName: 'Testing docstring validation script' - condition: true - - - script: | - source activate pandas-dev - cd asv_bench - asv check -E existing - git remote add upstream https://github.com/pandas-dev/pandas.git - git fetch upstream - if git diff upstream/master --name-only | grep -q "^asv_bench/"; then - asv machine --yes - ASV_OUTPUT="$(asv dev)" - if [[ $(echo "$ASV_OUTPUT" | grep "failed") ]]; then - echo "##vso[task.logissue type=error]Benchmarks run with errors" - echo "$ASV_OUTPUT" - exit 1 - else - echo "Benchmarks run without errors" - fi - else - echo "Benchmarks did not run, no changes detected" - fi - displayName: 'Running benchmarks' - condition: true - -- job: 'Web_and_Docs' - pool: - vmImage: ubuntu-16.04 - timeoutInMinutes: 90 - steps: - - script: | - echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' - echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' - displayName: 'Setting environment variables' - - - script: | - sudo apt-get install -y libc6-dev-i386 - ci/setup_env.sh - displayName: 'Setup environment and build pandas' - - - script: | - source activate pandas-dev - python web/pandas_web.py web/pandas --target-path=web/build - displayName: 'Build website' - - - script: | - source activate pandas-dev - # Next we should simply have `doc/make.py --warnings-are-errors`, everything else is required because the ipython directive doesn't fail the build on errors (https://github.com/ipython/ipython/issues/11547) - doc/make.py --warnings-are-errors | tee sphinx.log ; SPHINX_RET=${PIPESTATUS[0]} - grep -B1 "^<<<-------------------------------------------------------------------------$" sphinx.log ; IPY_RET=$(( $? != 1 )) - exit $(( $SPHINX_RET + $IPY_RET )) - displayName: 'Build documentation' - - - script: | - mkdir -p to_deploy/docs - cp -r web/build/* to_deploy/ - cp -r doc/build/html/* to_deploy/docs/ - displayName: 'Merge website and docs' - - - script: | - cd to_deploy - git init - touch .nojekyll - echo "dev.pandas.io" > CNAME - printf "User-agent: *\nDisallow: /" > robots.txt - git add --all . - git config user.email "pandas-dev@python.org" - git config user.name "pandas-bot" - git commit -m "pandas web and documentation in master" - displayName: 'Create git repo for docs build' - condition : | - and(not(eq(variables['Build.Reason'], 'PullRequest')), - eq(variables['Build.SourceBranch'], 'refs/heads/master')) - - # For `InstallSSHKey@0` to work, next steps are required: - # 1. Generate a pair of private/public keys (i.e. `ssh-keygen -t rsa -b 4096 -C "your_email@example.com"`) - # 2. Go to "Library > Secure files" in the Azure Pipelines dashboard: https://dev.azure.com/pandas-dev/pandas/_library?itemType=SecureFiles - # 3. Click on "+ Secure file" - # 4. Upload the private key (the name of the file must match with the specified in "sshKeySecureFile" input below, "pandas_docs_key") - # 5. Click on file name after it is created, tick the box "Authorize for use in all pipelines" and save - # 6. The public key specified in "sshPublicKey" is the pair of the uploaded private key, and needs to be set as a deploy key of the repo where the docs will be pushed (with write access): https://github.com/pandas-dev/pandas-dev.github.io/settings/keys - - task: InstallSSHKey@0 - inputs: - hostName: 'github.com,192.30.252.128 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' - sshPublicKey: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDHmz3l/EdqrgNxEUKkwDUuUcLv91unig03pYFGO/DMIgCmPdMG96zAgfnESd837Rm0wSSqylwSzkRJt5MV/TpFlcVifDLDQmUhqCeO8Z6dLl/oe35UKmyYICVwcvQTAaHNnYRpKC5IUlTh0JEtw9fGlnp1Ta7U1ENBLbKdpywczElhZu+hOQ892zqOj3CwA+U2329/d6cd7YnqIKoFN9DWT3kS5K6JE4IoBfQEVekIOs23bKjNLvPoOmi6CroAhu/K8j+NCWQjge5eJf2x/yTnIIP1PlEcXoHIr8io517posIx3TBup+CN8bNS1PpDW3jyD3ttl1uoBudjOQrobNnJeR6Rn67DRkG6IhSwr3BWj8alwUG5mTdZzwV5Pa9KZFdIiqX7NoDGg+itsR39QCn0thK8lGRNSR8KrWC1PSjecwelKBO7uQ7rnk/rkrZdBWR4oEA8YgNH8tirUw5WfOr5a0AIaJicKxGKNdMxZt+zmC+bS7F4YCOGIm9KHa43RrKhoGRhRf9fHHHKUPwFGqtWG4ykcUgoamDOURJyepesBAO3FiRE9rLU6ILbB3yEqqoekborHmAJD5vf7PWItW3Q/YQKuk3kkqRcKnexPyzyyq5lUgTi8CxxZdaASIOu294wjBhhdyHlXEkVTNJ9JKkj/obF+XiIIp0cBDsOXY9hDQ== pandas-dev@python.org' - sshKeySecureFile: 'pandas_docs_key' - displayName: 'Install GitHub ssh deployment key' - condition : | - and(not(eq(variables['Build.Reason'], 'PullRequest')), - eq(variables['Build.SourceBranch'], 'refs/heads/master')) - - - script: | - cd to_deploy - git remote add origin git@github.com:pandas-dev/pandas-dev.github.io.git - git push -f origin master - displayName: 'Publish web and docs to GitHub pages' - condition : | - and(not(eq(variables['Build.Reason'], 'PullRequest')), - eq(variables['Build.SourceBranch'], 'refs/heads/master')) From 611c2b89e92481763b6d2b6de800468b31959176 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 12:01:09 -0500 Subject: [PATCH 33/41] install using pip --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 07759325dbc1e..1365e6db50b14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,7 +69,8 @@ install: - cd .. - rm -rf numpy - python setup.py build_ext -inplace - - python setup.py install + - python -m pip install --no-build-isolation -e . + #- python setup.py install - python -c "import sys; print(sys.version_info)" - python -c "import pandas as pd" #- ci/setup_env.sh From eaa8af7d876e83cc1d2e5041864d2e03a42072d9 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 3 Oct 2019 11:51:51 -0500 Subject: [PATCH 34/41] troubleshoot --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1365e6db50b14..f16133d9d3cdc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,6 +73,7 @@ install: #- python setup.py install - python -c "import sys; print(sys.version_info)" - python -c "import pandas as pd" + - python -c "import hypothesis" #- ci/setup_env.sh - ci/submit_cython_cache.sh - echo "install done" @@ -88,7 +89,8 @@ install: script: - echo "script start" #- source activate pandas-dev - - ci/run_tests.sh + #- ci/run_tests.sh + - pytest pandas/tests --skip-slow after_script: - echo "after_script start" From 2d4351207199c88e9a99587333b1f4dd3c3c01b9 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 3 Oct 2019 12:45:39 -0500 Subject: [PATCH 35/41] working with just pytest, now try with run_tests --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f16133d9d3cdc..fc4844fb37763 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,8 +89,8 @@ install: script: - echo "script start" #- source activate pandas-dev - #- ci/run_tests.sh - - pytest pandas/tests --skip-slow + - ci/run_tests.sh + #- pytest pandas/tests --skip-slow after_script: - echo "after_script start" From ea5aba69e8f69ceedad5600559058db71c6be78f Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 3 Oct 2019 15:08:47 -0500 Subject: [PATCH 36/41] restore, try to do stuff conditionally --- .travis.yml | 72 ++++++++++++++++++++++++++++++--------------------- ci/build38.sh | 30 +++++++++++++++++++++ 2 files changed, 72 insertions(+), 30 deletions(-) create mode 100644 ci/build38.sh diff --git a/.travis.yml b/.travis.yml index fc4844fb37763..20b7332bab1f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,27 @@ matrix: env: - JOB="3.8-dev" PATTERN="(not slow and not network)" + - dist: trusty + env: + - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)" + + - dist: trusty + env: + - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" + + - dist: trusty + env: + - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true + + # In allow_failures + - dist: trusty + env: + - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" + + allow_failures: + - dist: trusty + env: + - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" before_install: - echo "before_install" @@ -56,41 +77,32 @@ before_install: # This overrides travis and tells it to look nowhere. - export BOTO_CONFIG=/dev/null + install: - - echo "install start" - - sudo apt-get install build-essential - - pip install --no-deps -U pip wheel setuptools - - pip install python-dateutil pytz cython pytest pytest-xdist hypothesis - - ci/prep_cython_cache.sh - - git clone https://github.com/numpy/numpy - - cd numpy - - python setup.py build_ext --inplace - - python setup.py install - - cd .. - - rm -rf numpy - - python setup.py build_ext -inplace - - python -m pip install --no-build-isolation -e . - #- python setup.py install - - python -c "import sys; print(sys.version_info)" - - python -c "import pandas as pd" - - python -c "import hypothesis" - #- ci/setup_env.sh - - ci/submit_cython_cache.sh - - echo "install done" - -#before_script: -# # display server (for clipboard functionality) needs to be started here, -# # does not work if done in install:setup_env.sh (GH-26103) -# - export DISPLAY=":99.0" -# - echo "sh -e /etc/init.d/xvfb start" -# - sh -e /etc/init.d/xvfb start -# - sleep 3 + if: env(JOB) = "3.8-dev" + - ci/build38.sh + if: env(JOB) != "3.8-dev" + - echo "install start" + - ci/prep_cython_cache.sh + - ci/setup_env.sh + - ci/submit_cython_cache.sh + - echo "install done" + + +before_script: + # display server (for clipboard functionality) needs to be started here, + # does not work if done in install:setup_env.sh (GH-26103) + if: env(JOB) != "3.8-dev" + - export DISPLAY=":99.0" + - echo "sh -e /etc/init.d/xvfb start" + - sh -e /etc/init.d/xvfb start + - sleep 3 script: - echo "script start" - #- source activate pandas-dev + if: env(JOB) != "3.8-dev" + - source activate pandas-dev - ci/run_tests.sh - #- pytest pandas/tests --skip-slow after_script: - echo "after_script start" diff --git a/ci/build38.sh b/ci/build38.sh new file mode 100644 index 0000000000000..b21b356c4df63 --- /dev/null +++ b/ci/build38.sh @@ -0,0 +1,30 @@ +#!/bin/bash -e +# Special build for python3.8 until numpy puts its own wheels up + +echo "python3.8 install start" + +sudo apt-get install build-essential +pip install --no-deps -U pip wheel setuptools +pip install python-dateutil pytz cython pytest pytest-xdist hypothesis + +ci/prep_cython_cache.sh + +git clone https://github.com/numpy/numpy +cd numpy +python setup.py build_ext --inplace +python setup.py install +cd .. +rm -rf numpy + +python setup.py build_ext -inplace +python -m pip install --no-build-isolation -e . + +python -c "import sys; print(sys.version_info)" +python -c "import pandas as pd" +python -c "import hypothesis" + +# TODO: Is there anything else in setup_env that we really want to do? +# ci/setup_env.sh + +ci/submit_cython_cache.sh +echo "python3.8 install done" From 620023a54722db44648f68a8c78fd7d58af71d49 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 3 Oct 2019 15:13:46 -0500 Subject: [PATCH 37/41] try to guess how conditional syntax works on travis --- .travis.yml | 30 +++++++++++++++--------------- ci/build38.sh | 7 ------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 20b7332bab1f2..d30ceadaa41d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,29 +79,29 @@ before_install: install: - if: env(JOB) = "3.8-dev" - - ci/build38.sh - if: env(JOB) != "3.8-dev" - - echo "install start" - - ci/prep_cython_cache.sh - - ci/setup_env.sh - - ci/submit_cython_cache.sh - - echo "install done" + - echo "install start" + - ci/prep_cython_cache.sh + - ci/build38.sh + if: env(JOB) = "3.8-dev" + - ci/setup_env.sh + if: env(JOB) != "3.8-dev" + - ci/submit_cython_cache.sh + - echo "install done" before_script: # display server (for clipboard functionality) needs to be started here, # does not work if done in install:setup_env.sh (GH-26103) - if: env(JOB) != "3.8-dev" - - export DISPLAY=":99.0" - - echo "sh -e /etc/init.d/xvfb start" - - sh -e /etc/init.d/xvfb start - - sleep 3 + - if: env(JOB) != "3.8-dev" + - export DISPLAY=":99.0" + - echo "sh -e /etc/init.d/xvfb start" + - sh -e /etc/init.d/xvfb start + - sleep 3 script: - echo "script start" - if: env(JOB) != "3.8-dev" - - source activate pandas-dev + - source activate pandas-dev + if: env(JOB) != "3.8-dev" - ci/run_tests.sh after_script: diff --git a/ci/build38.sh b/ci/build38.sh index b21b356c4df63..d72bce22a3cce 100644 --- a/ci/build38.sh +++ b/ci/build38.sh @@ -1,14 +1,10 @@ #!/bin/bash -e # Special build for python3.8 until numpy puts its own wheels up -echo "python3.8 install start" - sudo apt-get install build-essential pip install --no-deps -U pip wheel setuptools pip install python-dateutil pytz cython pytest pytest-xdist hypothesis -ci/prep_cython_cache.sh - git clone https://github.com/numpy/numpy cd numpy python setup.py build_ext --inplace @@ -25,6 +21,3 @@ python -c "import hypothesis" # TODO: Is there anything else in setup_env that we really want to do? # ci/setup_env.sh - -ci/submit_cython_cache.sh -echo "python3.8 install done" From e7cdbc617ef79fe82c2a647d9aded68f8165881d Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 3 Oct 2019 15:14:57 -0500 Subject: [PATCH 38/41] revert --- azure-pipelines.yml | 180 +++++++++++++++++++++++++++++++++++++++++ ci/deps/travis-38.yaml | 22 ----- ci/setup_env.sh | 4 +- 3 files changed, 182 insertions(+), 24 deletions(-) create mode 100644 azure-pipelines.yml delete mode 100644 ci/deps/travis-38.yaml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000000000..62c46b6970969 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,180 @@ +# Adapted from https://github.com/numba/numba/blob/master/azure-pipelines.yml +jobs: +# Mac and Linux use the same template +- template: ci/azure/posix.yml + parameters: + name: macOS + vmImage: xcode9-macos10.13 + +- template: ci/azure/posix.yml + parameters: + name: Linux + vmImage: ubuntu-16.04 + +- template: ci/azure/windows.yml + parameters: + name: Windows + vmImage: vs2017-win2016 + +- job: 'Checks' + pool: + vmImage: ubuntu-16.04 + timeoutInMinutes: 90 + steps: + - script: | + echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' + echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' + echo '##vso[task.setvariable variable=AZURE]true' + displayName: 'Setting environment variables' + + # Do not require a conda environment + - script: ci/code_checks.sh patterns + displayName: 'Looking for unwanted patterns' + condition: true + + - script: | + sudo apt-get install -y libc6-dev-i386 + ci/setup_env.sh + displayName: 'Setup environment and build pandas' + condition: true + + # Do not require pandas + - script: | + source activate pandas-dev + ci/code_checks.sh lint + displayName: 'Linting' + condition: true + + - script: | + source activate pandas-dev + ci/code_checks.sh dependencies + displayName: 'Dependencies consistency' + condition: true + + # Require pandas + - script: | + source activate pandas-dev + ci/code_checks.sh code + displayName: 'Checks on imported code' + condition: true + + - script: | + source activate pandas-dev + ci/code_checks.sh doctests + displayName: 'Running doctests' + condition: true + + - script: | + source activate pandas-dev + ci/code_checks.sh docstrings + displayName: 'Docstring validation' + condition: true + + - script: | + source activate pandas-dev + ci/code_checks.sh typing + displayName: 'Typing validation' + condition: true + + - script: | + source activate pandas-dev + pytest --capture=no --strict scripts + displayName: 'Testing docstring validation script' + condition: true + + - script: | + source activate pandas-dev + cd asv_bench + asv check -E existing + git remote add upstream https://github.com/pandas-dev/pandas.git + git fetch upstream + if git diff upstream/master --name-only | grep -q "^asv_bench/"; then + asv machine --yes + ASV_OUTPUT="$(asv dev)" + if [[ $(echo "$ASV_OUTPUT" | grep "failed") ]]; then + echo "##vso[task.logissue type=error]Benchmarks run with errors" + echo "$ASV_OUTPUT" + exit 1 + else + echo "Benchmarks run without errors" + fi + else + echo "Benchmarks did not run, no changes detected" + fi + displayName: 'Running benchmarks' + condition: true + +- job: 'Web_and_Docs' + pool: + vmImage: ubuntu-16.04 + timeoutInMinutes: 90 + steps: + - script: | + echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' + echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' + displayName: 'Setting environment variables' + + - script: | + sudo apt-get install -y libc6-dev-i386 + ci/setup_env.sh + displayName: 'Setup environment and build pandas' + + - script: | + source activate pandas-dev + python web/pandas_web.py web/pandas --target-path=web/build + displayName: 'Build website' + + - script: | + source activate pandas-dev + # Next we should simply have `doc/make.py --warnings-are-errors`, everything else is required because the ipython directive doesn't fail the build on errors (https://github.com/ipython/ipython/issues/11547) + doc/make.py --warnings-are-errors | tee sphinx.log ; SPHINX_RET=${PIPESTATUS[0]} + grep -B1 "^<<<-------------------------------------------------------------------------$" sphinx.log ; IPY_RET=$(( $? != 1 )) + exit $(( $SPHINX_RET + $IPY_RET )) + displayName: 'Build documentation' + + - script: | + mkdir -p to_deploy/docs + cp -r web/build/* to_deploy/ + cp -r doc/build/html/* to_deploy/docs/ + displayName: 'Merge website and docs' + + - script: | + cd to_deploy + git init + touch .nojekyll + echo "dev.pandas.io" > CNAME + printf "User-agent: *\nDisallow: /" > robots.txt + git add --all . + git config user.email "pandas-dev@python.org" + git config user.name "pandas-bot" + git commit -m "pandas web and documentation in master" + displayName: 'Create git repo for docs build' + condition : | + and(not(eq(variables['Build.Reason'], 'PullRequest')), + eq(variables['Build.SourceBranch'], 'refs/heads/master')) + + # For `InstallSSHKey@0` to work, next steps are required: + # 1. Generate a pair of private/public keys (i.e. `ssh-keygen -t rsa -b 4096 -C "your_email@example.com"`) + # 2. Go to "Library > Secure files" in the Azure Pipelines dashboard: https://dev.azure.com/pandas-dev/pandas/_library?itemType=SecureFiles + # 3. Click on "+ Secure file" + # 4. Upload the private key (the name of the file must match with the specified in "sshKeySecureFile" input below, "pandas_docs_key") + # 5. Click on file name after it is created, tick the box "Authorize for use in all pipelines" and save + # 6. The public key specified in "sshPublicKey" is the pair of the uploaded private key, and needs to be set as a deploy key of the repo where the docs will be pushed (with write access): https://github.com/pandas-dev/pandas-dev.github.io/settings/keys + - task: InstallSSHKey@0 + inputs: + hostName: 'github.com,192.30.252.128 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' + sshPublicKey: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDHmz3l/EdqrgNxEUKkwDUuUcLv91unig03pYFGO/DMIgCmPdMG96zAgfnESd837Rm0wSSqylwSzkRJt5MV/TpFlcVifDLDQmUhqCeO8Z6dLl/oe35UKmyYICVwcvQTAaHNnYRpKC5IUlTh0JEtw9fGlnp1Ta7U1ENBLbKdpywczElhZu+hOQ892zqOj3CwA+U2329/d6cd7YnqIKoFN9DWT3kS5K6JE4IoBfQEVekIOs23bKjNLvPoOmi6CroAhu/K8j+NCWQjge5eJf2x/yTnIIP1PlEcXoHIr8io517posIx3TBup+CN8bNS1PpDW3jyD3ttl1uoBudjOQrobNnJeR6Rn67DRkG6IhSwr3BWj8alwUG5mTdZzwV5Pa9KZFdIiqX7NoDGg+itsR39QCn0thK8lGRNSR8KrWC1PSjecwelKBO7uQ7rnk/rkrZdBWR4oEA8YgNH8tirUw5WfOr5a0AIaJicKxGKNdMxZt+zmC+bS7F4YCOGIm9KHa43RrKhoGRhRf9fHHHKUPwFGqtWG4ykcUgoamDOURJyepesBAO3FiRE9rLU6ILbB3yEqqoekborHmAJD5vf7PWItW3Q/YQKuk3kkqRcKnexPyzyyq5lUgTi8CxxZdaASIOu294wjBhhdyHlXEkVTNJ9JKkj/obF+XiIIp0cBDsOXY9hDQ== pandas-dev@python.org' + sshKeySecureFile: 'pandas_docs_key' + displayName: 'Install GitHub ssh deployment key' + condition : | + and(not(eq(variables['Build.Reason'], 'PullRequest')), + eq(variables['Build.SourceBranch'], 'refs/heads/master')) + + - script: | + cd to_deploy + git remote add origin git@github.com:pandas-dev/pandas-dev.github.io.git + git push -f origin master + displayName: 'Publish web and docs to GitHub pages' + condition : | + and(not(eq(variables['Build.Reason'], 'PullRequest')), + eq(variables['Build.SourceBranch'], 'refs/heads/master')) diff --git a/ci/deps/travis-38.yaml b/ci/deps/travis-38.yaml deleted file mode 100644 index e3d85a4e388b7..0000000000000 --- a/ci/deps/travis-38.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: pandas-dev -channels: - - defaults - - conda-forge - - c3i_test -dependencies: - #- python=3.8.* - - botocore>=1.11 - - cython>=0.29.13 - - python-dateutil - - nomkl - - pytz - # universal - - pytest>=5.0.0 - - pytest-xdist>=1.29.0 - - pytest-mock - - hypothesis>=3.58.0 - - s3fs<0.3 - - pip - - pyreadstat - - pip: - - moto diff --git a/ci/setup_env.sh b/ci/setup_env.sh index c245bbe9ed6a8..382491a947488 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -91,8 +91,8 @@ conda list conda remove --all -q -y -n pandas-dev echo -echo "conda env create -q" -time conda env create -q +echo "conda env create -q --file=${ENV_FILE}" +time conda env create -q --file="${ENV_FILE}" if [[ "$BITS32" == "yes" ]]; then From e8c579da2a491d9a77b2c2b376b1077c5754351f Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 9 Oct 2019 13:25:06 -0700 Subject: [PATCH 39/41] try it without conditionals in yaml --- .travis.yml | 7 +------ ci/build38.sh | 2 +- ci/setup_env.sh | 4 ++++ 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index d30ceadaa41d1..fec61bacbef08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,10 +81,7 @@ before_install: install: - echo "install start" - ci/prep_cython_cache.sh - - ci/build38.sh - if: env(JOB) = "3.8-dev" - ci/setup_env.sh - if: env(JOB) != "3.8-dev" - ci/submit_cython_cache.sh - echo "install done" @@ -92,7 +89,6 @@ install: before_script: # display server (for clipboard functionality) needs to be started here, # does not work if done in install:setup_env.sh (GH-26103) - - if: env(JOB) != "3.8-dev" - export DISPLAY=":99.0" - echo "sh -e /etc/init.d/xvfb start" - sh -e /etc/init.d/xvfb start @@ -100,8 +96,7 @@ before_script: script: - echo "script start" - - source activate pandas-dev - if: env(JOB) != "3.8-dev" + - if [ $JOB != "3.8-dev"]; then source activate pandas-dev; fi - ci/run_tests.sh after_script: diff --git a/ci/build38.sh b/ci/build38.sh index d72bce22a3cce..302839ebdba04 100644 --- a/ci/build38.sh +++ b/ci/build38.sh @@ -1,7 +1,7 @@ #!/bin/bash -e # Special build for python3.8 until numpy puts its own wheels up -sudo apt-get install build-essential +sudo apt-get install build-essential xvfb pip install --no-deps -U pip wheel setuptools pip install python-dateutil pytz cython pytest pytest-xdist hypothesis diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 382491a947488..be8c3645691fe 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -1,5 +1,9 @@ #!/bin/bash -e +if [ "$JOB" == "3.8-dev" ]; then + /bin/bash ci/build38.sh + exit 0 +fi # edit the locale file if needed if [ -n "$LOCALE_OVERRIDE" ]; then From a3c986b128b205cc45f65dd1cd8170ebee3d10f0 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 9 Oct 2019 14:35:52 -0700 Subject: [PATCH 40/41] troubleshoot --- .travis.yml | 3 ++- ci/build38.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index fec61bacbef08..aa1f616add79f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -96,7 +96,8 @@ before_script: script: - echo "script start" - - if [ $JOB != "3.8-dev"]; then source activate pandas-dev; fi + - echo "$JOB" + - if [ "$JOB" != "3.8-dev" ]; then source activate pandas-dev; fi - ci/run_tests.sh after_script: diff --git a/ci/build38.sh b/ci/build38.sh index 302839ebdba04..8f1c7dff1d24f 100644 --- a/ci/build38.sh +++ b/ci/build38.sh @@ -1,7 +1,7 @@ #!/bin/bash -e # Special build for python3.8 until numpy puts its own wheels up -sudo apt-get install build-essential xvfb +sudo apt-get install build-essential gcc xvfb pip install --no-deps -U pip wheel setuptools pip install python-dateutil pytz cython pytest pytest-xdist hypothesis From 8f4a80a8d9b8a35640bd0b7c88c867cbf2c020e4 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 9 Oct 2019 15:43:15 -0700 Subject: [PATCH 41/41] disable xvfb on 3.8-dev --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index aa1f616add79f..b9fa06304d387 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,7 +91,7 @@ before_script: # does not work if done in install:setup_env.sh (GH-26103) - export DISPLAY=":99.0" - echo "sh -e /etc/init.d/xvfb start" - - sh -e /etc/init.d/xvfb start + - if [ "$JOB" != "3.8-dev" ]; then sh -e /etc/init.d/xvfb start; fi - sleep 3 script: