Duplicate PR of #22854 (testing publishing docs from azure)

.travis.yml

@@ -52,7 +52,7 @@ matrix:
           - python-gtk2
     - dist: trusty
       env:
-        - JOB="3.6, lint, coverage" ENV_FILE="ci/deps/travis-36.yaml" PATTERN="not slow and not network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true LINT=true
+        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36.yaml" PATTERN="not slow and not network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true
     - dist: trusty
       env:
         - JOB="3.7, NumPy dev" ENV_FILE="ci/deps/travis-37-numpydev.yaml" PATTERN="not slow and not network" TEST_ARGS="-W error" PANDAS_TESTING_MODE="deprecate"
@@ -108,7 +108,6 @@ script:
   - source activate pandas-dev
   - ci/run_build_docs.sh
   - ci/run_tests.sh
-  - ci/code_checks.sh
 
 after_script:
   - echo "after_script start"

azure-pipelines.yml

@@ -23,3 +23,123 @@ jobs:
   parameters:
     name: WindowsPy27
     vmImage: vs2017-win2016
+
+- job: 'Checks_and_doc'
+  pool:
+    vmImage: ubuntu-16.04
+  timeoutInMinutes: 90
+  steps:
+  - script: |
+      # XXX next command should avoid redefining the path in every step, but
+      # made the process crash as it couldn't find deactivate
+      #echo '##vso[task.prependpath]$HOME/miniconda3/bin'
+      echo '##vso[task.setvariable variable=CONDA_ENV]pandas-dev'
+      echo '##vso[task.setvariable variable=ENV_FILE]environment.yml'
+      echo '##vso[task.setvariable variable=AZURE]true'
+    displayName: 'Setting environment variables'
+
+  # Do not require a conda environment
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      ci/code_checks.sh patterns
+    displayName: 'Looking for unwanted patterns'
+    condition: true
+
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      sudo apt-get install -y libc6-dev-i386
+      ci/incremental/install_miniconda.sh
+      ci/incremental/setup_conda_environment.sh
+    displayName: 'Set up environment'
+
+  # Do not require pandas
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      source activate pandas-dev
+      ci/code_checks.sh lint
+    displayName: 'Linting'
+    condition: true
+
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      source activate pandas-dev
+      ci/code_checks.sh dependencies
+    displayName: 'Dependencies consistency'
+    condition: true
+
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      source activate pandas-dev
+      ci/incremental/build.sh
+    displayName: 'Build'
+    condition: true
+
+  # Require pandas
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      source activate pandas-dev
+      ci/code_checks.sh code
+    displayName: 'Checks on imported code'
+    condition: true
+
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      source activate pandas-dev
+      ci/code_checks.sh doctests
+    displayName: 'Running doctests'
+    condition: true
+
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      source activate pandas-dev
+      ci/code_checks.sh docstrings
+    displayName: 'Docstring validation'
+    condition: true
+
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      source activate pandas-dev
+      pytest --capture=no --strict scripts
+    displayName: 'Testing docstring validaton script'
+    condition: true
+
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      source activate pandas-dev
+      doc/make.py html --num-jobs=2
+    displayName: 'Building docs'
+    condition: true
+
+  - script: |
+      az extension add --name storage-preview
+      az storage blob upload-batch --connection-string $CONNECTION_STRING \
+                                   --source $SOURCE \
+                                   --destination '$web'
+      echo "Documentation uploaded to https://pandas.blob.core.windows.net"
+    displayName: 'Publishing docs (Azure storage)'
+    condition: true
+    env:
+      CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING)
+      SOURCE: $(Build.SourcesDirectory)/doc/build/html/
+
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      source activate pandas-dev
+      git remote add upstream https://github.com/pandas-dev/pandas.git
+      git fetch upstream
+      if git diff upstream/master --name-only | grep -q "^asv_bench/"; then
+          cd asv_bench
+          asv machine --yes
+          ASV_OUTPUT="$(asv dev)"
+          if [[ $(echo "$ASV_OUTPUT" | grep "failed") ]]; then
+              echo "##vso[task.logissue type=error]Benchmarks run with errors"
+              echo $ASV_OUTPUT
+              exit 1
+          else
+              echo "Benchmarks run without errors"
+          fi
+      else
+          echo "Benchmarks did not run, no changes detected"
+      fi
+    displayName: 'Running benchmarks'
+    condition: true

ci/code_checks.sh

@@ -5,25 +5,48 @@
 # This script is intended for both the CI and to check locally that code standards are
 # respected. We are currently linting (PEP-8 and similar), looking for patterns of
 # common mistakes (sphinx directives with missing blank lines, old style classes,
-# unwanted imports...), and we also run doctests here (currently some files only).
-# In the future we may want to add the validation of docstrings and other checks here.
+# unwanted imports...), we run doctests here (currently some files only), and we
+# validate formatting error in docstrings.
 #
 # Usage:
 #   $ ./ci/code_checks.sh               # run all checks
 #   $ ./ci/code_checks.sh lint          # run linting only
 #   $ ./ci/code_checks.sh patterns      # check for patterns that should not exist
+#   $ ./ci/code_checks.sh code          # checks on imported code
 #   $ ./ci/code_checks.sh doctests      # run doctests
+#   $ ./ci/code_checks.sh docstrings    # validate docstring errors
 #   $ ./ci/code_checks.sh dependencies  # check that dependencies are consistent
 
-echo "inside $0"
-[[ $LINT ]] || { echo "NOT Linting. To lint use: LINT=true $0 $1"; exit 0; }
-[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "doctests" || "$1" == "dependencies"  ]] \
-    || { echo "Unknown command $1. Usage: $0 [lint|patterns|doctests|dependencies]"; exit 9999; }
+[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "dependencies" ]] || \
+    { echo "Unknown command $1. Usage: $0 [lint|patterns|code|doctests|docstrings|dependencies]"; exit 9999; }
 
 BASE_DIR="$(dirname $0)/.."
 RET=0
 CHECK=$1
 
+function invgrep {
+    # grep with inverse exist status and formatting for azure-pipelines
+    #
+    # This function works exactly as grep, but with opposite exit status:
+    # - 0 (success) when no patterns are found
+    # - 1 (fail) when the patterns are found
+    #
+    # This is useful for the CI, as we want to fail if one of the patterns
+    # that we want to avoid is found by grep.
+    if [[ "$AZURE" == "true" ]]; then
+        set -o pipefail
+        grep -n "$@" | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Found unwanted pattern: " $3}'
+    else
+        grep "$@"
+    fi
+    return $((! $?))
+}
+
+if [[ "$AZURE" == "true" ]]; then
+    FLAKE8_FORMAT="##vso[task.logissue type=error;sourcepath=%(path)s;linenumber=%(row)s;columnnumber=%(col)s;code=%(code)s;]%(text)s"
+else
+    FLAKE8_FORMAT="default"
+fi
 
 ### LINTING ###
 if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
@@ -35,30 +58,30 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
 
     # pandas/_libs/src is C code, so no need to search there.
     MSG='Linting .py code' ; echo $MSG
-    flake8 .
+    flake8 --format="$FLAKE8_FORMAT" .
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Linting .pyx code' ; echo $MSG
-    flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411
+    flake8 --format="$FLAKE8_FORMAT" pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Linting .pxd and .pxi.in' ; echo $MSG
-    flake8 pandas/_libs --filename=*.pxi.in,*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403
+    flake8 --format="$FLAKE8_FORMAT" pandas/_libs --filename=*.pxi.in,*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     echo "flake8-rst --version"
     flake8-rst --version
 
     MSG='Linting code-blocks in .rst documentation' ; echo $MSG
-    flake8-rst doc/source --filename=*.rst
+    flake8-rst doc/source --filename=*.rst --format="$FLAKE8_FORMAT"
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     # Check that cython casting is of the form `<type>obj` as opposed to `<type> obj`;
     # it doesn't make a difference, but we want to be internally consistent.
     # Note: this grep pattern is (intended to be) equivalent to the python
     # regex r'(?<![ ->])> '
     MSG='Linting .pyx code for spacing conventions in casting' ; echo $MSG
-    ! grep -r -E --include '*.pyx' --include '*.pxi.in' '[a-zA-Z0-9*]> ' pandas/_libs
+    invgrep -r -E --include '*.pyx' --include '*.pxi.in' '[a-zA-Z0-9*]> ' pandas/_libs
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     # readability/casting: Warnings about C casting instead of C++ casting
@@ -88,43 +111,48 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
 
     # Check for imports from pandas.core.common instead of `import pandas.core.common as com`
     MSG='Check for non-standard imports' ; echo $MSG
-    ! grep -R --include="*.py*" -E "from pandas.core.common import " pandas
+    invgrep -R --include="*.py*" -E "from pandas.core.common import " pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check for pytest warns' ; echo $MSG
-    ! grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/
+    invgrep -r -E --include '*.py' 'pytest\.warns' pandas/tests/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     # Check for the following code in testing: `np.testing` and `np.array_equal`
     MSG='Check for invalid testing' ; echo $MSG
-    ! grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/
+    invgrep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal`
     MSG='Check for invalid EA testing' ; echo $MSG
-    ! grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base
+    invgrep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check for deprecated messages without sphinx directive' ; echo $MSG
-    ! grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas
+    invgrep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check for old-style classes' ; echo $MSG
-    ! grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts
+    invgrep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check for backticks incorrectly rendering because of missing spaces' ; echo $MSG
-    ! grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/
+    invgrep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check for incorrect sphinx directives' ; echo $MSG
-    ! grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas ./doc/source
+    invgrep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas ./doc/source
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check that the deprecated `assert_raises_regex` is not used (`pytest.raises(match=pattern)` should be used instead)' ; echo $MSG
     ! grep -R --exclude=*.pyc --exclude=testing.py --exclude=test_testing.py assert_raises_regex pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+fi
+
+### CODE ###
+if [[ -z "$CHECK" || "$CHECK" == "code" ]]; then
+
     MSG='Check for modules that pandas should not import' ; echo $MSG
     python -c "
 import sys
@@ -135,7 +163,7 @@ blacklist = {'bs4', 'gcsfs', 'html5lib', 'ipython', 'jinja2' 'hypothesis',
              'tables', 'xlrd', 'xlsxwriter', 'xlwt'}
 mods = blacklist & set(m.split('.')[0] for m in sys.modules)
 if mods:
-    sys.stderr.write('pandas should not import: {}\n'.format(', '.join(mods)))
+    sys.stderr.write('err: pandas should not import: {}\n'.format(', '.join(mods)))
     sys.exit(len(mods))
     "
     RET=$(($RET + $?)) ; echo $MSG "DONE"
@@ -157,7 +185,7 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
 
     MSG='Doctests generic.py' ; echo $MSG
     pytest -q --doctest-modules pandas/core/generic.py \
-        -k"-_set_axis_name -_xs -describe -droplevel -groupby -interpolate -pct_change -pipe -reindex -reindex_axis -to_json -transpose -values -xs"
+        -k"-_set_axis_name -_xs -describe -droplevel -groupby -interpolate -pct_change -pipe -reindex -reindex_axis -to_json -transpose -values -xs -to_clipboard"
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Doctests top-level reshaping functions' ; echo $MSG
@@ -178,11 +206,22 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
 
 fi
 
+### DOCSTRINGS ###
+if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
+
+    MSG='Validate docstrings (GL06, SS04, PR03, PR05, EX04)' ; echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=azure --errors=GL06,SS04,PR03,PR05,EX04
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+fi
+
 ### DEPENDENCIES ###
 if [[ -z "$CHECK" || "$CHECK" == "dependencies" ]]; then
+
     MSG='Check that requirements-dev.txt has been generated from environment.yml' ; echo $MSG
-    $BASE_DIR/scripts/generate_pip_deps_from_conda.py --compare
+    $BASE_DIR/scripts/generate_pip_deps_from_conda.py --compare --azure
     RET=$(($RET + $?)) ; echo $MSG "DONE"
+
 fi
 
 exit $RET

ci/deps/travis-36.yaml

@@ -7,16 +7,9 @@ dependencies:
   - cython>=0.28.2
   - dask
   - fastparquet
-  - flake8>=3.5
-  - flake8-comprehensions
-  - flake8-rst=0.4.2
   - gcsfs
   - geopandas
   - html5lib
-  - ipython
-  - isort
-  - jinja2
-  - lxml
   - matplotlib
   - nomkl
   - numexpr
@@ -32,7 +25,6 @@ dependencies:
   - s3fs
   - scikit-learn
   - scipy
-  - seaborn
   - sqlalchemy
   - statsmodels
   - xarray
@@ -48,6 +40,5 @@ dependencies:
   - pip:
     - brotlipy
     - coverage
-    - cpplint
     - pandas-datareader
     - python-dateutil

ci/incremental/build.sh

@@ -4,7 +4,7 @@
 set -v -e
 
 echo "[building extensions]"
-python setup.py build_ext -q --inplace
+python setup.py build_ext -q --inplace -j2
 python -m pip install -e .
 
 echo