From 46620f259a62e5947ef8725d6c1b41ceb2600f68 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Sep 2018 14:21:52 +0100 Subject: [PATCH 001/104] Moving one-line linting and checks from lint.sh to azure steps --- azure-pipelines.yml | 55 +++++++++++++++++++++ ci/lint.sh | 118 -------------------------------------------- 2 files changed, 55 insertions(+), 118 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c82dafa224961..42d616fdf3ac3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,3 +23,58 @@ jobs: parameters: name: WindowsPy27 vmImage: vs2017-win2017 + +- job: 'Linting' + pool: + vmImage: ubuntu-16.04 + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + architecture: 'x64' + # We're ignoring the following codes across the board + #E402, # module level import not at top of file + #E731, # do not assign a lambda expression, use a def + #E741, # do not use variables named 'l', 'O', or 'I' + #W503, # line break before binary operator + #C406, # Unnecessary (list/tuple) literal - rewrite as a dict literal. + #C408, # Unnecessary (dict/list/tuple) call - rewrite as a literal. + #C409, # Unnecessary (list/tuple) passed to tuple() - (remove the outer call to tuple()/rewrite as a tuple literal). + #C410 # Unnecessary (list/tuple) passed to list() - (remove the outer call to list()/rewrite as a list literal). + + # pandas/_libs/src is C code, so no need to search there. + - script: flake8 pandas --filename=*.py --exclude pandas/_libs/src --ignore=C406,C408,C409,E402,E731,E741,W503 + displayName: 'Linting *.py code' + - script: flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411 + displayName: 'Linting *.pyx code' + - script: cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/tslibs/src/datetime + displayName: 'Linting pandas/_libs/tslibs/src/datetime' + - script: flake8 setup.py --ignore=E402,E731,E741,W503 + displayName: 'Linting setup.py' + - script: flake8 scripts --filename=*.py --ignore=C408,E402,E731,E741,W503 + displayName: 'Linting scripts' + # TODO I think flake8 the scripts directory already checks the scripts/tests + # will remove later + - script: flake8 scripts/tests --filename=*.py + displayName: 'Linting scripts tests' + - script: flake8 asv_bench/benchmarks/ --exclude=asv_bench/benchmarks/*.py --ignore=F811,C406,C408,C409,C410 + displayName: 'Linting asv benchmarks' + - script: flake8 doc/make.py doc/source/conf.py --ignore=E402,E731,E741,W503 + displayName: 'Linting doc scripts' + # Check for imports from pandas.core.common instead of `import pandas.core.common as com` + - script: grep -R --include="*.py*" -E "from pandas.core.common import " pandas + displayName: 'Check for non-standard imports' + - script: grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ + displayName: 'Check for pytest warns' + # Check for the following code in testing: `np.testing` and `np.array_equal` + - script: grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ + displayName: 'Check for invalid testing' + # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal` + - script: grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base + displayName: 'Check for invalid EA testing' + - script: grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas + displayName: 'Check for deprecated messages without sphinx directive' + - script: grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts + displayName: 'Check for old-style classes' + - script: grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ + displayName: 'Check for backticks incorrectly rendering because of missing spaces' diff --git a/ci/lint.sh b/ci/lint.sh index 533e1d18d8e0e..8713991b3f5b4 100755 --- a/ci/lint.sh +++ b/ci/lint.sh @@ -18,54 +18,6 @@ if [ "$LINT" ]; then #C409, # Unnecessary (list/tuple) passed to tuple() - (remove the outer call to tuple()/rewrite as a tuple literal). #C410 # Unnecessary (list/tuple) passed to list() - (remove the outer call to list()/rewrite as a list literal). - # pandas/_libs/src is C code, so no need to search there. - echo "Linting *.py" - flake8 pandas --filename=*.py --exclude pandas/_libs/src --ignore=C406,C408,C409,E402,E731,E741,W503 - if [ $? -ne "0" ]; then - RET=1 - fi - - flake8 scripts/tests --filename=*.py - if [ $? -ne "0" ]; then - RET=1 - fi - echo "Linting *.py DONE" - - echo "Linting setup.py" - flake8 setup.py --ignore=E402,E731,E741,W503 - if [ $? -ne "0" ]; then - RET=1 - fi - echo "Linting setup.py DONE" - - echo "Linting asv_bench/benchmarks/" - flake8 asv_bench/benchmarks/ --exclude=asv_bench/benchmarks/*.py --ignore=F811,C406,C408,C409,C410 - if [ $? -ne "0" ]; then - RET=1 - fi - echo "Linting asv_bench/benchmarks/*.py DONE" - - echo "Linting scripts/*.py" - flake8 scripts --filename=*.py --ignore=C408,E402,E731,E741,W503 - if [ $? -ne "0" ]; then - RET=1 - fi - echo "Linting scripts/*.py DONE" - - echo "Linting doc scripts" - flake8 doc/make.py doc/source/conf.py --ignore=E402,E731,E741,W503 - if [ $? -ne "0" ]; then - RET=1 - fi - echo "Linting doc scripts DONE" - - echo "Linting *.pyx" - flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411 - if [ $? -ne "0" ]; then - RET=1 - fi - echo "Linting *.pyx DONE" - echo "Linting *.pxi.in" for path in 'src' do @@ -105,51 +57,6 @@ if [ "$LINT" ]; then RET=1 fi done - echo "linting -> pandas/_libs/tslibs/src/datetime" - cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/tslibs/src/datetime - if [ $? -ne "0" ]; then - RET=1 - fi - echo "Linting *.c and *.h DONE" - - echo "Check for invalid testing" - - # Check for the following code in testing: - # - # np.testing - # np.array_equal - grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ - - if [ $? = "0" ]; then - RET=1 - fi - - # Check for pytest.warns - grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ - - if [ $? = "0" ]; then - RET=1 - fi - - # Check for the following code in the extension array base tests - # tm.assert_frame_equal - # tm.assert_series_equal - grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base - - if [ $? = "0" ]; then - RET=1 - fi - - echo "Check for invalid testing DONE" - - # Check for imports from pandas.core.common instead - # of `import pandas.core.common as com` - echo "Check for non-standard imports" - grep -R --include="*.py*" -E "from pandas.core.common import " pandas - if [ $? = "0" ]; then - RET=1 - fi - echo "Check for non-standard imports DONE" echo "Check for incorrect sphinx directives" SPHINX_DIRECTIVES=$(echo \ @@ -164,31 +71,6 @@ if [ "$LINT" ]; then fi done echo "Check for incorrect sphinx directives DONE" - - echo "Check for deprecated messages without sphinx directive" - grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas - - if [ $? = "0" ]; then - RET=1 - fi - echo "Check for deprecated messages without sphinx directive DONE" - - echo "Check for old-style classes" - grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts - - if [ $? = "0" ]; then - RET=1 - fi - echo "Check for old-style classes DONE" - - echo "Check for backticks incorrectly rendering because of missing spaces" - grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ - - if [ $? = "0" ]; then - RET=1 - fi - echo "Check for backticks incorrectly rendering because of missing spaces DONE" - else echo "NOT Linting" fi From dc8c528a8fed5e91c03b4b7dd4f01c87417f4022 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Sep 2018 15:47:10 +0100 Subject: [PATCH 002/104] Moving everything from lint.sh to azure (so far repeating the commands that used to be loops). Installing missing packages, splitting linting and checks, and continuing on errors --- .travis.yml | 3 +- azure-pipelines.yml | 58 ++++++++++++++++++++++++++++++--- ci/lint.sh | 78 --------------------------------------------- 3 files changed, 55 insertions(+), 84 deletions(-) delete mode 100755 ci/lint.sh diff --git a/.travis.yml b/.travis.yml index 40baee2c03ea0..8addc0002133f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,7 +45,7 @@ matrix: - language-pack-zh-hans - dist: trusty env: - - JOB="2.7, lint" ENV_FILE="ci/travis-27.yaml" TEST_ARGS="--skip-slow" LINT=true + - JOB="2.7" ENV_FILE="ci/travis-27.yaml" TEST_ARGS="--skip-slow" addons: apt: packages: @@ -114,7 +114,6 @@ script: - ci/run_build_docs.sh - ci/script_single.sh - ci/script_multi.sh - - ci/lint.sh - ci/doctests.sh - echo "checking imports" - source activate pandas && python ci/check_imports.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 42d616fdf3ac3..f04b327193884 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -25,6 +25,7 @@ jobs: vmImage: vs2017-win2017 - job: 'Linting' + continueOnError: true pool: vmImage: ubuntu-16.04 steps: @@ -32,6 +33,9 @@ jobs: inputs: versionSpec: '3.7' architecture: 'x64' + - script: python -m pip install flake8 + displayName: 'Installing flake8' + continueOnError: false # We're ignoring the following codes across the board #E402, # module level import not at top of file #E731, # do not assign a lambda expression, use a def @@ -47,20 +51,54 @@ jobs: displayName: 'Linting *.py code' - script: flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411 displayName: 'Linting *.pyx code' - - script: cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/tslibs/src/datetime - displayName: 'Linting pandas/_libs/tslibs/src/datetime' + - script: flake8 pandas/src --filename=*.pxi.in --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 + displayName: 'Linting *.pxi.in' + - script: flake8 pandas/_libs --filename=*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 + displayName: 'Linting *.pxd' - script: flake8 setup.py --ignore=E402,E731,E741,W503 displayName: 'Linting setup.py' - script: flake8 scripts --filename=*.py --ignore=C408,E402,E731,E741,W503 displayName: 'Linting scripts' # TODO I think flake8 the scripts directory already checks the scripts/tests # will remove later - - script: flake8 scripts/tests --filename=*.py - displayName: 'Linting scripts tests' + #- script: flake8 scripts/tests --filename=*.py + # displayName: 'Linting scripts tests' - script: flake8 asv_bench/benchmarks/ --exclude=asv_bench/benchmarks/*.py --ignore=F811,C406,C408,C409,C410 displayName: 'Linting asv benchmarks' - script: flake8 doc/make.py doc/source/conf.py --ignore=E402,E731,E741,W503 displayName: 'Linting doc scripts' + + - script: python -m pip install cpplint + displayName: 'Installing cpplint' + continueOnError: false + # readability/casting: Warnings about C casting instead of C++ casting + # runtime/int: Warnings about using C number types instead of C++ ones + # build/include_subdir: Warnings about prefacing included header files with directory + + # We don't lint all C files because we don't want to lint any that are built + # from Cython files nor do we want to lint C files that we didn't modify for + # this particular codebase (e.g. src/headers, src/klib, src/msgpack). However, + # we can lint all header files since they aren't "generated" like C files are. + # TODO the next 3 commands are the same changing just the path, we should avoid repeating + # them, and have kind of loop thing + - script: cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/src/*.h + displayName: 'Linting *.c and *.h (*.h)' + - script: cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/src/parser + displayName: 'Linting *.c and *.h (parser)' + - script: cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/src/ujson + displayName: 'Linting *.c and *.h (ujson)' + - script: cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/tslibs/src/datetime + displayName: 'Linting pandas/_libs/tslibs/src/datetime' + +- job: 'Checks' + continueOnError: true + pool: + vmImage: ubuntu-16.04 + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + architecture: 'x64' # Check for imports from pandas.core.common instead of `import pandas.core.common as com` - script: grep -R --include="*.py*" -E "from pandas.core.common import " pandas displayName: 'Check for non-standard imports' @@ -78,3 +116,15 @@ jobs: displayName: 'Check for old-style classes' - script: grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ displayName: 'Check for backticks incorrectly rendering because of missing spaces' + # TODO the next 2 commands are the same changing just the path, we should avoid repeating + # them, and have kind of loop thing + # TODO SPHINX_DIRECTIVES should be defined as a variable, as it was in lint.sh. Not + # sure yet how to define variables with the output of a sctipt, will take care of it later + # SPHINX_DIRECTIVES=$(echo \ + # "autosummary|contents|currentmodule|deprecated|function|image|"\ + # "important|include|ipython|literalinclude|math|module|note|raw|"\ + # "seealso|toctree|versionadded|versionchanged|warning" | tr -d "[:space:]") + - script: grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas + displayName: 'Check for incorrect sphinx directives (./pandas)' + - script: grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./doc/source + displayName: 'Check for incorrect sphinx directives (./doc/source)' diff --git a/ci/lint.sh b/ci/lint.sh deleted file mode 100755 index 8713991b3f5b4..0000000000000 --- a/ci/lint.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -echo "inside $0" - -source activate pandas - -RET=0 - -if [ "$LINT" ]; then - - # We're ignoring the following codes across the board - #E402, # module level import not at top of file - #E731, # do not assign a lambda expression, use a def - #E741, # do not use variables named 'l', 'O', or 'I' - #W503, # line break before binary operator - #C406, # Unnecessary (list/tuple) literal - rewrite as a dict literal. - #C408, # Unnecessary (dict/list/tuple) call - rewrite as a literal. - #C409, # Unnecessary (list/tuple) passed to tuple() - (remove the outer call to tuple()/rewrite as a tuple literal). - #C410 # Unnecessary (list/tuple) passed to list() - (remove the outer call to list()/rewrite as a list literal). - - echo "Linting *.pxi.in" - for path in 'src' - do - echo "linting -> pandas/$path" - flake8 pandas/$path --filename=*.pxi.in --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 - if [ $? -ne "0" ]; then - RET=1 - fi - done - echo "Linting *.pxi.in DONE" - - echo "Linting *.pxd" - for path in '_libs' - do - echo "linting -> pandas/$path" - flake8 pandas/$path --filename=*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 - if [ $? -ne "0" ]; then - RET=1 - fi - done - echo "Linting *.pxd DONE" - - # readability/casting: Warnings about C casting instead of C++ casting - # runtime/int: Warnings about using C number types instead of C++ ones - # build/include_subdir: Warnings about prefacing included header files with directory - - # We don't lint all C files because we don't want to lint any that are built - # from Cython files nor do we want to lint C files that we didn't modify for - # this particular codebase (e.g. src/headers, src/klib, src/msgpack). However, - # we can lint all header files since they aren't "generated" like C files are. - echo "Linting *.c and *.h" - for path in '*.h' 'parser' 'ujson' - do - echo "linting -> pandas/_libs/src/$path" - cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/src/$path - if [ $? -ne "0" ]; then - RET=1 - fi - done - - echo "Check for incorrect sphinx directives" - SPHINX_DIRECTIVES=$(echo \ - "autosummary|contents|currentmodule|deprecated|function|image|"\ - "important|include|ipython|literalinclude|math|module|note|raw|"\ - "seealso|toctree|versionadded|versionchanged|warning" | tr -d "[:space:]") - for path in './pandas' './doc/source' - do - grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. ($SPHINX_DIRECTIVES):[^:]" $path - if [ $? = "0" ]; then - RET=1 - fi - done - echo "Check for incorrect sphinx directives DONE" -else - echo "NOT Linting" -fi - -exit $RET From adeb0caeffa07ca188a7130130e1cc95c574eefc Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Sep 2018 16:15:08 +0100 Subject: [PATCH 003/104] Fixing continueOnError settings --- azure-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f04b327193884..5f68ef6a9eb93 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -25,7 +25,7 @@ jobs: vmImage: vs2017-win2017 - job: 'Linting' - continueOnError: true + continueOnError: 'true' pool: vmImage: ubuntu-16.04 steps: @@ -35,7 +35,7 @@ jobs: architecture: 'x64' - script: python -m pip install flake8 displayName: 'Installing flake8' - continueOnError: false + continueOnError: 'false' # We're ignoring the following codes across the board #E402, # module level import not at top of file #E731, # do not assign a lambda expression, use a def @@ -70,7 +70,7 @@ jobs: - script: python -m pip install cpplint displayName: 'Installing cpplint' - continueOnError: false + continueOnError: 'false' # readability/casting: Warnings about C casting instead of C++ casting # runtime/int: Warnings about using C number types instead of C++ ones # build/include_subdir: Warnings about prefacing included header files with directory @@ -91,7 +91,7 @@ jobs: displayName: 'Linting pandas/_libs/tslibs/src/datetime' - job: 'Checks' - continueOnError: true + continueOnError: 'true' pool: vmImage: ubuntu-16.04 steps: From dcfb2039a74360b3b805e04ff7107c9816949663 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Sep 2018 16:28:18 +0100 Subject: [PATCH 004/104] Reverting the exit status of the grep commands, we want them to fail when lines are found (exit 0) --- azure-pipelines.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5f68ef6a9eb93..29ad1a0aac593 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -100,21 +100,21 @@ jobs: versionSpec: '3.7' architecture: 'x64' # Check for imports from pandas.core.common instead of `import pandas.core.common as com` - - script: grep -R --include="*.py*" -E "from pandas.core.common import " pandas + - script: ! grep -R --include="*.py*" -E "from pandas.core.common import " pandas displayName: 'Check for non-standard imports' - - script: grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ + - script: ! grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ displayName: 'Check for pytest warns' # Check for the following code in testing: `np.testing` and `np.array_equal` - - script: grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ + - script: ! grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ displayName: 'Check for invalid testing' # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal` - - script: grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base + - script: ! grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base displayName: 'Check for invalid EA testing' - - script: grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas + - script: ! grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas displayName: 'Check for deprecated messages without sphinx directive' - - script: grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts + - script: ! grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts displayName: 'Check for old-style classes' - - script: grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ + - script: ! grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ displayName: 'Check for backticks incorrectly rendering because of missing spaces' # TODO the next 2 commands are the same changing just the path, we should avoid repeating # them, and have kind of loop thing @@ -124,7 +124,7 @@ jobs: # "autosummary|contents|currentmodule|deprecated|function|image|"\ # "important|include|ipython|literalinclude|math|module|note|raw|"\ # "seealso|toctree|versionadded|versionchanged|warning" | tr -d "[:space:]") - - script: grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas + - script: ! grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas displayName: 'Check for incorrect sphinx directives (./pandas)' - - script: grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./doc/source + - script: ! grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./doc/source displayName: 'Check for incorrect sphinx directives (./doc/source)' From b765df80d7c703e43a6315fdc54a174f44923d92 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Sep 2018 16:29:49 +0100 Subject: [PATCH 005/104] Testing the continueOnError, doesn't seem to be working at job level, with or without quotes --- azure-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 29ad1a0aac593..c5865885cbd64 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -49,8 +49,10 @@ jobs: # pandas/_libs/src is C code, so no need to search there. - script: flake8 pandas --filename=*.py --exclude pandas/_libs/src --ignore=C406,C408,C409,E402,E731,E741,W503 displayName: 'Linting *.py code' + continueOnError: 'true' - script: flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411 displayName: 'Linting *.pyx code' + continueOnError: true - script: flake8 pandas/src --filename=*.pxi.in --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 displayName: 'Linting *.pxi.in' - script: flake8 pandas/_libs --filename=*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 From 3c77fc946377100e8772742bde1760d0baa71582 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Sep 2018 16:33:01 +0100 Subject: [PATCH 006/104] Escaping exclamation marks to reverse the exist status of grep, azure seems to fail because of it --- azure-pipelines.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c5865885cbd64..ecda227a425d5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -102,21 +102,21 @@ jobs: versionSpec: '3.7' architecture: 'x64' # Check for imports from pandas.core.common instead of `import pandas.core.common as com` - - script: ! grep -R --include="*.py*" -E "from pandas.core.common import " pandas + - script: \! grep -R --include="*.py*" -E "from pandas.core.common import " pandas displayName: 'Check for non-standard imports' - - script: ! grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ + - script: \! grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ displayName: 'Check for pytest warns' # Check for the following code in testing: `np.testing` and `np.array_equal` - - script: ! grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ + - script: \! grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ displayName: 'Check for invalid testing' # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal` - - script: ! grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base + - script: \! grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base displayName: 'Check for invalid EA testing' - - script: ! grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas + - script: \! grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas displayName: 'Check for deprecated messages without sphinx directive' - - script: ! grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts + - script: \! grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts displayName: 'Check for old-style classes' - - script: ! grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ + - script: \! grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ displayName: 'Check for backticks incorrectly rendering because of missing spaces' # TODO the next 2 commands are the same changing just the path, we should avoid repeating # them, and have kind of loop thing @@ -126,7 +126,7 @@ jobs: # "autosummary|contents|currentmodule|deprecated|function|image|"\ # "important|include|ipython|literalinclude|math|module|note|raw|"\ # "seealso|toctree|versionadded|versionchanged|warning" | tr -d "[:space:]") - - script: ! grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas + - script: \! grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas displayName: 'Check for incorrect sphinx directives (./pandas)' - - script: ! grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./doc/source + - script: \! grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./doc/source displayName: 'Check for incorrect sphinx directives (./doc/source)' From 3d8af7d1f44b56c4dd52c395f9d9c17eeec6a882 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Sep 2018 16:51:32 +0100 Subject: [PATCH 007/104] Adding the continueOnError to every step, and trying if bash instead of script solves the exclamation mark problems --- azure-pipelines.yml | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ecda227a425d5..e1c4d423c53cc 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -25,7 +25,7 @@ jobs: vmImage: vs2017-win2017 - job: 'Linting' - continueOnError: 'true' + continueOnError: 'true' # XXX this doesn't seem to be working, specifying in each step pool: vmImage: ubuntu-16.04 steps: @@ -55,18 +55,23 @@ jobs: continueOnError: true - script: flake8 pandas/src --filename=*.pxi.in --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 displayName: 'Linting *.pxi.in' + continueOnError: 'true' - script: flake8 pandas/_libs --filename=*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 displayName: 'Linting *.pxd' + continueOnError: 'true' - script: flake8 setup.py --ignore=E402,E731,E741,W503 displayName: 'Linting setup.py' + continueOnError: 'true' - script: flake8 scripts --filename=*.py --ignore=C408,E402,E731,E741,W503 displayName: 'Linting scripts' + continueOnError: 'true' # TODO I think flake8 the scripts directory already checks the scripts/tests # will remove later #- script: flake8 scripts/tests --filename=*.py # displayName: 'Linting scripts tests' - script: flake8 asv_bench/benchmarks/ --exclude=asv_bench/benchmarks/*.py --ignore=F811,C406,C408,C409,C410 displayName: 'Linting asv benchmarks' + continueOnError: 'true' - script: flake8 doc/make.py doc/source/conf.py --ignore=E402,E731,E741,W503 displayName: 'Linting doc scripts' @@ -85,15 +90,18 @@ jobs: # them, and have kind of loop thing - script: cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/src/*.h displayName: 'Linting *.c and *.h (*.h)' + continueOnError: 'true' - script: cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/src/parser displayName: 'Linting *.c and *.h (parser)' + continueOnError: 'true' - script: cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/src/ujson displayName: 'Linting *.c and *.h (ujson)' + continueOnError: 'true' - script: cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/tslibs/src/datetime displayName: 'Linting pandas/_libs/tslibs/src/datetime' - job: 'Checks' - continueOnError: 'true' + continueOnError: 'true' # XXX this doesn't seem to be working, specifying in each step pool: vmImage: ubuntu-16.04 steps: @@ -102,22 +110,29 @@ jobs: versionSpec: '3.7' architecture: 'x64' # Check for imports from pandas.core.common instead of `import pandas.core.common as com` - - script: \! grep -R --include="*.py*" -E "from pandas.core.common import " pandas + - bash: \! grep -R --include="*.py*" -E "from pandas.core.common import " pandas displayName: 'Check for non-standard imports' - - script: \! grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ + continueOnError: 'true' + - bash: \! grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ displayName: 'Check for pytest warns' + continueOnError: 'true' # Check for the following code in testing: `np.testing` and `np.array_equal` - - script: \! grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ + - bash: \! grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ displayName: 'Check for invalid testing' + continueOnError: 'true' # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal` - - script: \! grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base + - bash: \! grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base displayName: 'Check for invalid EA testing' - - script: \! grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas + continueOnError: 'true' + - bash: \! grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas displayName: 'Check for deprecated messages without sphinx directive' - - script: \! grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts + continueOnError: 'true' + - bash: \! grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts displayName: 'Check for old-style classes' - - script: \! grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ + continueOnError: 'true' + - bash: \! grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ displayName: 'Check for backticks incorrectly rendering because of missing spaces' + continueOnError: 'true' # TODO the next 2 commands are the same changing just the path, we should avoid repeating # them, and have kind of loop thing # TODO SPHINX_DIRECTIVES should be defined as a variable, as it was in lint.sh. Not @@ -126,7 +141,8 @@ jobs: # "autosummary|contents|currentmodule|deprecated|function|image|"\ # "important|include|ipython|literalinclude|math|module|note|raw|"\ # "seealso|toctree|versionadded|versionchanged|warning" | tr -d "[:space:]") - - script: \! grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas + - bash: \! grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas displayName: 'Check for incorrect sphinx directives (./pandas)' - - script: \! grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./doc/source + continueOnError: 'true' + - bash: \! grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./doc/source displayName: 'Check for incorrect sphinx directives (./doc/source)' From ed1064d19540b19a167014b9b8f20489e042b8b2 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Sep 2018 17:01:58 +0100 Subject: [PATCH 008/104] Fixes to missing continueOnError, and to the grep command exit status --- azure-pipelines.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e1c4d423c53cc..f2c35406728b2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -74,6 +74,7 @@ jobs: continueOnError: 'true' - script: flake8 doc/make.py doc/source/conf.py --ignore=E402,E731,E741,W503 displayName: 'Linting doc scripts' + continueOnError: 'true' - script: python -m pip install cpplint displayName: 'Installing cpplint' @@ -110,27 +111,27 @@ jobs: versionSpec: '3.7' architecture: 'x64' # Check for imports from pandas.core.common instead of `import pandas.core.common as com` - - bash: \! grep -R --include="*.py*" -E "from pandas.core.common import " pandas + - bash: grep -R --include="*.py*" -E "from pandas.core.common import " pandas || echo 'Check OK' displayName: 'Check for non-standard imports' continueOnError: 'true' - - bash: \! grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ + - bash: grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ || echo 'Check OK' displayName: 'Check for pytest warns' continueOnError: 'true' # Check for the following code in testing: `np.testing` and `np.array_equal` - - bash: \! grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ + - bash: grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ || echo 'Check OK' displayName: 'Check for invalid testing' continueOnError: 'true' # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal` - - bash: \! grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base + - bash: grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base || echo 'Check OK' displayName: 'Check for invalid EA testing' continueOnError: 'true' - - bash: \! grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas + - bash: grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas || echo 'Check OK' displayName: 'Check for deprecated messages without sphinx directive' continueOnError: 'true' - - bash: \! grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts + - bash: grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts || echo 'Check OK' displayName: 'Check for old-style classes' continueOnError: 'true' - - bash: \! grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ + - bash: grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ || echo 'Check OK' displayName: 'Check for backticks incorrectly rendering because of missing spaces' continueOnError: 'true' # TODO the next 2 commands are the same changing just the path, we should avoid repeating @@ -141,8 +142,8 @@ jobs: # "autosummary|contents|currentmodule|deprecated|function|image|"\ # "important|include|ipython|literalinclude|math|module|note|raw|"\ # "seealso|toctree|versionadded|versionchanged|warning" | tr -d "[:space:]") - - bash: \! grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas + - bash: grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas || echo 'Check OK' displayName: 'Check for incorrect sphinx directives (./pandas)' continueOnError: 'true' - - bash: \! grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./doc/source + - bash: grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./doc/source || echo 'Check OK' displayName: 'Check for incorrect sphinx directives (./doc/source)' From 8cb7b0f86f38a202519881e916a500cb1fbe66fa Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Sep 2018 20:05:36 +0100 Subject: [PATCH 009/104] Testing couple of ways to invert the exit status of grep --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f2c35406728b2..d172d52d60033 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -111,10 +111,10 @@ jobs: versionSpec: '3.7' architecture: 'x64' # Check for imports from pandas.core.common instead of `import pandas.core.common as com` - - bash: grep -R --include="*.py*" -E "from pandas.core.common import " pandas || echo 'Check OK' + - bash: grep -R --include="*.py*" -E "from pandas.core.common import " pandas ; [ $? == 0 ] displayName: 'Check for non-standard imports' continueOnError: 'true' - - bash: grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ || echo 'Check OK' + - script: "! grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/" displayName: 'Check for pytest warns' continueOnError: 'true' # Check for the following code in testing: `np.testing` and `np.array_equal` From dee5a08c0b14e9ab92881cb76770057c4d327092 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Sep 2018 20:11:15 +0100 Subject: [PATCH 010/104] Fixing azure config syntax error --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d172d52d60033..28d682f5c3c9f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -114,7 +114,7 @@ jobs: - bash: grep -R --include="*.py*" -E "from pandas.core.common import " pandas ; [ $? == 0 ] displayName: 'Check for non-standard imports' continueOnError: 'true' - - script: "! grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/" + - script: grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ displayName: 'Check for pytest warns' continueOnError: 'true' # Check for the following code in testing: `np.testing` and `np.array_equal` From 54f9e9836547e69a17d106143e6dde316d79dc6d Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Sep 2018 20:17:49 +0100 Subject: [PATCH 011/104] Fixing the checks (they should be ok now), and adding an intentional pep8 issue in scripts/tests --- azure-pipelines.yml | 16 ++++++++-------- scripts/tests/test_validate_docstrings.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 28d682f5c3c9f..9cb7c263163c8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -111,27 +111,27 @@ jobs: versionSpec: '3.7' architecture: 'x64' # Check for imports from pandas.core.common instead of `import pandas.core.common as com` - - bash: grep -R --include="*.py*" -E "from pandas.core.common import " pandas ; [ $? == 0 ] + - script: grep -R --include="*.py*" -E "from pandas.core.common import " pandas ; [ $? != 0 ] displayName: 'Check for non-standard imports' continueOnError: 'true' - script: grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ displayName: 'Check for pytest warns' continueOnError: 'true' # Check for the following code in testing: `np.testing` and `np.array_equal` - - bash: grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ || echo 'Check OK' + - script: grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ ; [ $? != 0 ] displayName: 'Check for invalid testing' continueOnError: 'true' # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal` - - bash: grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base || echo 'Check OK' + - script: grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base ; [ $? != 0 ] displayName: 'Check for invalid EA testing' continueOnError: 'true' - - bash: grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas || echo 'Check OK' + - script: grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas ; [ $? != 0 ] displayName: 'Check for deprecated messages without sphinx directive' continueOnError: 'true' - - bash: grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts || echo 'Check OK' + - script: grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts ; [ $? != 0 ] displayName: 'Check for old-style classes' continueOnError: 'true' - - bash: grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ || echo 'Check OK' + - script: grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ ; [ $? != 0 ] displayName: 'Check for backticks incorrectly rendering because of missing spaces' continueOnError: 'true' # TODO the next 2 commands are the same changing just the path, we should avoid repeating @@ -142,8 +142,8 @@ jobs: # "autosummary|contents|currentmodule|deprecated|function|image|"\ # "important|include|ipython|literalinclude|math|module|note|raw|"\ # "seealso|toctree|versionadded|versionchanged|warning" | tr -d "[:space:]") - - bash: grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas || echo 'Check OK' + - script: grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas ; [ $? != 0 ] displayName: 'Check for incorrect sphinx directives (./pandas)' continueOnError: 'true' - - bash: grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./doc/source || echo 'Check OK' + - script: grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./doc/source ; [ $? != 0 ] displayName: 'Check for incorrect sphinx directives (./doc/source)' diff --git a/scripts/tests/test_validate_docstrings.py b/scripts/tests/test_validate_docstrings.py index 00496f771570b..a182218537286 100644 --- a/scripts/tests/test_validate_docstrings.py +++ b/scripts/tests/test_validate_docstrings.py @@ -7,7 +7,7 @@ validate_one = validate_docstrings.validate_one from pandas.util.testing import capture_stderr - +# FIXME adding this line, and making it longer than 80 characters, so the linting fail, and it shows whether this directory is really being tested twice class GoodDocStrings(object): """ From 20cb360d68b6c84c7332f3c069c0540f26193126 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Sep 2018 20:23:14 +0100 Subject: [PATCH 012/104] Removing intentional pep8 issue, and fixing last known problems --- azure-pipelines.yml | 6 +----- scripts/tests/test_validate_docstrings.py | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9cb7c263163c8..adc485b58272c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -65,10 +65,6 @@ jobs: - script: flake8 scripts --filename=*.py --ignore=C408,E402,E731,E741,W503 displayName: 'Linting scripts' continueOnError: 'true' - # TODO I think flake8 the scripts directory already checks the scripts/tests - # will remove later - #- script: flake8 scripts/tests --filename=*.py - # displayName: 'Linting scripts tests' - script: flake8 asv_bench/benchmarks/ --exclude=asv_bench/benchmarks/*.py --ignore=F811,C406,C408,C409,C410 displayName: 'Linting asv benchmarks' continueOnError: 'true' @@ -114,7 +110,7 @@ jobs: - script: grep -R --include="*.py*" -E "from pandas.core.common import " pandas ; [ $? != 0 ] displayName: 'Check for non-standard imports' continueOnError: 'true' - - script: grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ + - script: grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ ; [ $? != 0 ] displayName: 'Check for pytest warns' continueOnError: 'true' # Check for the following code in testing: `np.testing` and `np.array_equal` diff --git a/scripts/tests/test_validate_docstrings.py b/scripts/tests/test_validate_docstrings.py index a182218537286..00496f771570b 100644 --- a/scripts/tests/test_validate_docstrings.py +++ b/scripts/tests/test_validate_docstrings.py @@ -7,7 +7,7 @@ validate_one = validate_docstrings.validate_one from pandas.util.testing import capture_stderr -# FIXME adding this line, and making it longer than 80 characters, so the linting fail, and it shows whether this directory is really being tested twice + class GoodDocStrings(object): """ From 50d275785980edb23346914d7b185ffeaa6c9d9f Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Sep 2018 21:48:01 +0100 Subject: [PATCH 013/104] fixing style of azure settings --- azure-pipelines.yml | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index adc485b58272c..c136bf6b9c238 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -53,11 +53,8 @@ jobs: - script: flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411 displayName: 'Linting *.pyx code' continueOnError: true - - script: flake8 pandas/src --filename=*.pxi.in --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 - displayName: 'Linting *.pxi.in' - continueOnError: 'true' - - script: flake8 pandas/_libs --filename=*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 - displayName: 'Linting *.pxd' + - script: flake8 pandas/_libs --filename=*.pxi.in,*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 + displayName: 'Linting *.pxd and *.pxi.in' continueOnError: 'true' - script: flake8 setup.py --ignore=E402,E731,E741,W503 displayName: 'Linting setup.py' @@ -65,7 +62,7 @@ jobs: - script: flake8 scripts --filename=*.py --ignore=C408,E402,E731,E741,W503 displayName: 'Linting scripts' continueOnError: 'true' - - script: flake8 asv_bench/benchmarks/ --exclude=asv_bench/benchmarks/*.py --ignore=F811,C406,C408,C409,C410 + - script: flake8 asv_bench/benchmarks/ --exclude=asv_bench/benchmarks/*.py --ignore=F811,C406,C408,C409,C410 displayName: 'Linting asv benchmarks' continueOnError: 'true' - script: flake8 doc/make.py doc/source/conf.py --ignore=E402,E731,E741,W503 @@ -85,17 +82,13 @@ jobs: # we can lint all header files since they aren't "generated" like C files are. # TODO the next 3 commands are the same changing just the path, we should avoid repeating # them, and have kind of loop thing - - script: cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/src/*.h - displayName: 'Linting *.c and *.h (*.h)' - continueOnError: 'true' - - script: cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/src/parser - displayName: 'Linting *.c and *.h (parser)' - continueOnError: 'true' - - script: cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/src/ujson - displayName: 'Linting *.c and *.h (ujson)' - continueOnError: 'true' - - script: cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/tslibs/src/datetime - displayName: 'Linting pandas/_libs/tslibs/src/datetime' + - script: | + cpplint --quiet --extensions=c,h --headers=h --recursive + --filter=-readability/casting,-runtime/int,-build/include_subdir + pandas/_libs/src/*.h + pandas/_libs/src/parser pandas/_libs/ujson + pandas/_libs/tslibs/src/datetime + displayName: 'Linting *.c and *.h' - job: 'Checks' continueOnError: 'true' # XXX this doesn't seem to be working, specifying in each step @@ -130,16 +123,16 @@ jobs: - script: grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ ; [ $? != 0 ] displayName: 'Check for backticks incorrectly rendering because of missing spaces' continueOnError: 'true' - # TODO the next 2 commands are the same changing just the path, we should avoid repeating - # them, and have kind of loop thing # TODO SPHINX_DIRECTIVES should be defined as a variable, as it was in lint.sh. Not # sure yet how to define variables with the output of a sctipt, will take care of it later # SPHINX_DIRECTIVES=$(echo \ # "autosummary|contents|currentmodule|deprecated|function|image|"\ # "important|include|ipython|literalinclude|math|module|note|raw|"\ # "seealso|toctree|versionadded|versionchanged|warning" | tr -d "[:space:]") - - script: grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas ; [ $? != 0 ] - displayName: 'Check for incorrect sphinx directives (./pandas)' - continueOnError: 'true' - - script: grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./doc/source ; [ $? != 0 ] - displayName: 'Check for incorrect sphinx directives (./doc/source)' + - script: | + grep -R --include="*.py" --include="*.pyx" --include="*.rst" + -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" + ./pandas + ./doc/source + ; [ $? != 0 ] + displayName: 'Check for incorrect sphinx directives' From 19a213c9af797634a6eeec93ec6bddff1221a8d7 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 29 Sep 2018 10:49:44 +0100 Subject: [PATCH 014/104] Replacing continueOnError (that converts errors in warnings) to condition, which runs the task, but doesn't change the outcome of the build --- azure-pipelines.yml | 61 +++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c136bf6b9c238..9f5a3e14fd9dc 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -35,43 +35,32 @@ jobs: architecture: 'x64' - script: python -m pip install flake8 displayName: 'Installing flake8' - continueOnError: 'false' - # We're ignoring the following codes across the board - #E402, # module level import not at top of file - #E731, # do not assign a lambda expression, use a def - #E741, # do not use variables named 'l', 'O', or 'I' - #W503, # line break before binary operator - #C406, # Unnecessary (list/tuple) literal - rewrite as a dict literal. - #C408, # Unnecessary (dict/list/tuple) call - rewrite as a literal. - #C409, # Unnecessary (list/tuple) passed to tuple() - (remove the outer call to tuple()/rewrite as a tuple literal). - #C410 # Unnecessary (list/tuple) passed to list() - (remove the outer call to list()/rewrite as a list literal). - # pandas/_libs/src is C code, so no need to search there. - - script: flake8 pandas --filename=*.py --exclude pandas/_libs/src --ignore=C406,C408,C409,E402,E731,E741,W503 + - script: flake8 pandas --exclude pandas/_libs/src displayName: 'Linting *.py code' - continueOnError: 'true' - - script: flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411 - displayName: 'Linting *.pyx code' - continueOnError: true - - script: flake8 pandas/_libs --filename=*.pxi.in,*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 - displayName: 'Linting *.pxd and *.pxi.in' - continueOnError: 'true' - - script: flake8 setup.py --ignore=E402,E731,E741,W503 + condition: true + - script: flake8 setup.py displayName: 'Linting setup.py' - continueOnError: 'true' - - script: flake8 scripts --filename=*.py --ignore=C408,E402,E731,E741,W503 + condition: true + - script: flake8 scripts displayName: 'Linting scripts' - continueOnError: 'true' - - script: flake8 asv_bench/benchmarks/ --exclude=asv_bench/benchmarks/*.py --ignore=F811,C406,C408,C409,C410 + condition: true + - script: flake8 asv_bench/benchmarks/ --exclude=asv_bench/benchmarks/*.py displayName: 'Linting asv benchmarks' - continueOnError: 'true' - - script: flake8 doc/make.py doc/source/conf.py --ignore=E402,E731,E741,W503 + condition: true + - script: flake8 doc/make.py doc/source/conf.py displayName: 'Linting doc scripts' - continueOnError: 'true' + condition: true + - script: flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411 + displayName: 'Linting *.pyx code' + condition: true + - script: flake8 pandas/_libs --filename=*.pxi.in,*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 + displayName: 'Linting *.pxd and *.pxi.in' + condition: true - script: python -m pip install cpplint displayName: 'Installing cpplint' - continueOnError: 'false' + condition: true # readability/casting: Warnings about C casting instead of C++ casting # runtime/int: Warnings about using C number types instead of C++ ones # build/include_subdir: Warnings about prefacing included header files with directory @@ -89,6 +78,7 @@ jobs: pandas/_libs/src/parser pandas/_libs/ujson pandas/_libs/tslibs/src/datetime displayName: 'Linting *.c and *.h' + condition: true - job: 'Checks' continueOnError: 'true' # XXX this doesn't seem to be working, specifying in each step @@ -102,27 +92,27 @@ jobs: # Check for imports from pandas.core.common instead of `import pandas.core.common as com` - script: grep -R --include="*.py*" -E "from pandas.core.common import " pandas ; [ $? != 0 ] displayName: 'Check for non-standard imports' - continueOnError: 'true' + condition: true - script: grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ ; [ $? != 0 ] displayName: 'Check for pytest warns' - continueOnError: 'true' + condition: true # Check for the following code in testing: `np.testing` and `np.array_equal` - script: grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ ; [ $? != 0 ] displayName: 'Check for invalid testing' - continueOnError: 'true' + condition: true # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal` - script: grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base ; [ $? != 0 ] displayName: 'Check for invalid EA testing' - continueOnError: 'true' + condition: true - script: grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas ; [ $? != 0 ] displayName: 'Check for deprecated messages without sphinx directive' - continueOnError: 'true' + condition: true - script: grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts ; [ $? != 0 ] displayName: 'Check for old-style classes' - continueOnError: 'true' + condition: true - script: grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ ; [ $? != 0 ] displayName: 'Check for backticks incorrectly rendering because of missing spaces' - continueOnError: 'true' + condition: true # TODO SPHINX_DIRECTIVES should be defined as a variable, as it was in lint.sh. Not # sure yet how to define variables with the output of a sctipt, will take care of it later # SPHINX_DIRECTIVES=$(echo \ @@ -136,3 +126,4 @@ jobs: ./doc/source ; [ $? != 0 ] displayName: 'Check for incorrect sphinx directives' + condition: true From 0cf5da96922d04c23313c1db3f5c5ccd402acd36 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 29 Sep 2018 10:56:48 +0100 Subject: [PATCH 015/104] Fixing multiline in yaml, and removing job level continueOnError --- azure-pipelines.yml | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9f5a3e14fd9dc..73c19e66bf472 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -25,7 +25,6 @@ jobs: vmImage: vs2017-win2017 - job: 'Linting' - continueOnError: 'true' # XXX this doesn't seem to be working, specifying in each step pool: vmImage: ubuntu-16.04 steps: @@ -69,19 +68,16 @@ jobs: # from Cython files nor do we want to lint C files that we didn't modify for # this particular codebase (e.g. src/headers, src/klib, src/msgpack). However, # we can lint all header files since they aren't "generated" like C files are. - # TODO the next 3 commands are the same changing just the path, we should avoid repeating - # them, and have kind of loop thing - - script: | - cpplint --quiet --extensions=c,h --headers=h --recursive - --filter=-readability/casting,-runtime/int,-build/include_subdir - pandas/_libs/src/*.h - pandas/_libs/src/parser pandas/_libs/ujson - pandas/_libs/tslibs/src/datetime + - script: > + cpplint --quiet --extensions=c,h --headers=h --recursive + --filter=-readability/casting,-runtime/int,-build/include_subdir + pandas/_libs/src/*.h + pandas/_libs/src/parser pandas/_libs/ujson + pandas/_libs/tslibs/src/datetime displayName: 'Linting *.c and *.h' condition: true - job: 'Checks' - continueOnError: 'true' # XXX this doesn't seem to be working, specifying in each step pool: vmImage: ubuntu-16.04 steps: @@ -119,11 +115,11 @@ jobs: # "autosummary|contents|currentmodule|deprecated|function|image|"\ # "important|include|ipython|literalinclude|math|module|note|raw|"\ # "seealso|toctree|versionadded|versionchanged|warning" | tr -d "[:space:]") - - script: | - grep -R --include="*.py" --include="*.pyx" --include="*.rst" - -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" - ./pandas - ./doc/source - ; [ $? != 0 ] + - script: > + grep -R --include="*.py" --include="*.pyx" --include="*.rst" + -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" + ./pandas + ./doc/source + ; [ $? != 0 ] displayName: 'Check for incorrect sphinx directives' condition: true From 4d1070257cd74a2e026e5b8734baad09c3ff0da7 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 30 Sep 2018 19:10:12 +0100 Subject: [PATCH 016/104] Changing the format of flake8 output to azure (so it creates links, and it shows them without having to enter the log) --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index e4a2357def474..5d15d9d08c1f1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,6 +22,7 @@ ignore = C409, # Unnecessary (list/tuple) passed to tuple() - (remove the outer call to tuple()/rewrite as a tuple literal). C410 # Unnecessary (list/tuple) passed to list() - (remove the outer call to list()/rewrite as a list literal). max-line-length = 79 +format = "##vso[task.logissue type=error;sourcepath=%(path)s;linenumber=%(row)s;columnnumber=%(col)s;code=%(code)s;]%(text)s" [yapf] based_on_style = pep8 From a356f03f64b1aef7fb561b015e1a1107d2d731d9 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 30 Sep 2018 19:22:35 +0100 Subject: [PATCH 017/104] Removing unnecessary quotes in flake8 format config --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 5d15d9d08c1f1..55c00ccea00f5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,7 +22,7 @@ ignore = C409, # Unnecessary (list/tuple) passed to tuple() - (remove the outer call to tuple()/rewrite as a tuple literal). C410 # Unnecessary (list/tuple) passed to list() - (remove the outer call to list()/rewrite as a list literal). max-line-length = 79 -format = "##vso[task.logissue type=error;sourcepath=%(path)s;linenumber=%(row)s;columnnumber=%(col)s;code=%(code)s;]%(text)s" +format = ##vso[task.logissue type=error;sourcepath=%(path)s;linenumber=%(row)s;columnnumber=%(col)s;code=%(code)s;]%(text)s [yapf] based_on_style = pep8 From 50cb86709ee16cf37179dcbe60fef55f1422d83e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 30 Sep 2018 20:01:23 +0100 Subject: [PATCH 018/104] flake8 format breaks pip (ConfigParser) when present in setup.cfg, moving to the command for now --- azure-pipelines.yml | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 73c19e66bf472..c90b9eed94f67 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -35,7 +35,7 @@ jobs: - script: python -m pip install flake8 displayName: 'Installing flake8' # pandas/_libs/src is C code, so no need to search there. - - script: flake8 pandas --exclude pandas/_libs/src + - script: flake8 pandas --exclude pandas/_libs/src --format="##vso[task.logissue type=error;sourcepath=%(path)s;linenumber=%(row)s;columnnumber=%(col)s;code=%(code)s;]%(text)s" displayName: 'Linting *.py code' condition: true - script: flake8 setup.py diff --git a/setup.cfg b/setup.cfg index 55c00ccea00f5..a9dccfab55a54 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,7 +22,7 @@ ignore = C409, # Unnecessary (list/tuple) passed to tuple() - (remove the outer call to tuple()/rewrite as a tuple literal). C410 # Unnecessary (list/tuple) passed to list() - (remove the outer call to list()/rewrite as a list literal). max-line-length = 79 -format = ##vso[task.logissue type=error;sourcepath=%(path)s;linenumber=%(row)s;columnnumber=%(col)s;code=%(code)s;]%(text)s +#format = ##vso[task.logissue type=error;sourcepath=%(path)s;linenumber=%(row)s;columnnumber=%(col)s;code=%(code)s;]%(text)s [yapf] based_on_style = pep8 From 091193ca426ac94c68766a9c0b0aad74c8b192bc Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 30 Sep 2018 20:15:04 +0100 Subject: [PATCH 019/104] Adding azure formatting to all failing flake8 --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c90b9eed94f67..a7f3d81c9cb1b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -47,13 +47,13 @@ jobs: - script: flake8 asv_bench/benchmarks/ --exclude=asv_bench/benchmarks/*.py displayName: 'Linting asv benchmarks' condition: true - - script: flake8 doc/make.py doc/source/conf.py + - script: flake8 doc/make.py doc/source/conf.py --format="##vso[task.logissue type=error;sourcepath=%(path)s;linenumber=%(row)s;columnnumber=%(col)s;code=%(code)s;]%(text)s" displayName: 'Linting doc scripts' condition: true - script: flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411 displayName: 'Linting *.pyx code' condition: true - - script: flake8 pandas/_libs --filename=*.pxi.in,*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 + - script: flake8 pandas/_libs --filename=*.pxi.in,*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 --format="##vso[task.logissue type=error;sourcepath=%(path)s;linenumber=%(row)s;columnnumber=%(col)s;code=%(code)s;]%(text)s" displayName: 'Linting *.pxd and *.pxi.in' condition: true From e7c276aaca9cbbaba1e1f135185a107ce1b4e2c1 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 10 Oct 2018 23:48:19 +0530 Subject: [PATCH 020/104] Fixing azure job name --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ff6abe6dfe01e..fda67034a43df 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,7 +24,7 @@ jobs: name: WindowsPy27 vmImage: vs2017-win2016 -- job: 'Code checks' +- job: 'Code_checks' pool: vmImage: ubuntu-16.04 steps: From 8f73c087a3297483e2b4aa605536f917142bb1ca Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 12 Oct 2018 17:18:16 +0530 Subject: [PATCH 021/104] Making the format look good for flake8 and patterns, and adding file with errors to see how they look --- ci/code_checks.sh | 49 +++++++++++++++++++++++++++++++++-------------- pandas/pep.py | 19 ++++++++++++++++++ 2 files changed, 54 insertions(+), 14 deletions(-) create mode 100644 pandas/pep.py diff --git a/ci/code_checks.sh b/ci/code_checks.sh index eced3bf34e7c6..79fce525414a0 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -14,14 +14,35 @@ # $ ./ci/code_checks.sh patterns # check for patterns that should not exist # $ ./ci/code_checks.sh doctests # run doctests -echo "inside $0" -[[ $LINT ]] || { echo "NOT Linting. To lint use: LINT=true $0 $1"; exit 0; } [[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "doctests" ]] || { echo "Unkown command $1. Usage: $0 [lint|patterns|doctests]"; exit 9999; } source activate pandas RET=0 CHECK=$1 +function invgrep { + # grep with inverse exist status and formatting for azure-pipelines + # + # This function works exactly as grep, but with opposite exit status: + # - 0 (success) when no patterns are found + # - 1 (fail) when the patterns are found + # + # This is useful for the CI, as we want to fail if one of the patterns + # that we want to avoid is found by grep. + if [[ "$AZURE" == "true" ]]; then + set -o pipefail + grep -n "$@" | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Found unwanted pattern: " $3}' + else + grep "$@" + fi + return $((! $?)) +} + +if [[ "$AZURE" == "true" ]]; then + FLAKE8_FORMAT="##vso[task.logissue type=error;sourcepath=%(path)s;linenumber=%(row)s;columnnumber=%(col)s;code=%(code)s;]%(text)s" +else + FLAKE8_FORMAT="default" +fi ### LINTING ### if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then @@ -33,15 +54,15 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then # pandas/_libs/src is C code, so no need to search there. MSG='Linting .py code' ; echo $MSG - flake8 . + flake8 --format="$FLAKE8_FORMAT" . RET=$(($RET + $?)) ; echo $MSG "DONE" MSG='Linting .pyx code' ; echo $MSG - flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411 + flake8 --format="$FLAKE8_FORMAT" pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411 RET=$(($RET + $?)) ; echo $MSG "DONE" MSG='Linting .pxd and .pxi.in' ; echo $MSG - flake8 pandas/_libs --filename=*.pxi.in,*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 + flake8 --format="$FLAKE8_FORMAT" pandas/_libs --filename=*.pxi.in,*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 RET=$(($RET + $?)) ; echo $MSG "DONE" # readability/casting: Warnings about C casting instead of C++ casting @@ -63,37 +84,37 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then # Check for imports from pandas.core.common instead of `import pandas.core.common as com` MSG='Check for non-standard imports' ; echo $MSG - ! grep -R --include="*.py*" -E "from pandas.core.common import " pandas + invgrep -R --include="*.py*" -E "from pandas.core.common import " pandas RET=$(($RET + $?)) ; echo $MSG "DONE" MSG='Check for pytest warns' ; echo $MSG - ! grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ + invgrep -r -E --include '*.py' 'pytest\.warns' pandas/tests/ RET=$(($RET + $?)) ; echo $MSG "DONE" # Check for the following code in testing: `np.testing` and `np.array_equal` MSG='Check for invalid testing' ; echo $MSG - ! grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ + invgrep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ RET=$(($RET + $?)) ; echo $MSG "DONE" # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal` MSG='Check for invalid EA testing' ; echo $MSG - ! grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base + invgrep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base RET=$(($RET + $?)) ; echo $MSG "DONE" MSG='Check for deprecated messages without sphinx directive' ; echo $MSG - ! grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas + invgrep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas RET=$(($RET + $?)) ; echo $MSG "DONE" MSG='Check for old-style classes' ; echo $MSG - ! grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts + invgrep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts RET=$(($RET + $?)) ; echo $MSG "DONE" MSG='Check for backticks incorrectly rendering because of missing spaces' ; echo $MSG - ! grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ + invgrep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ RET=$(($RET + $?)) ; echo $MSG "DONE" MSG='Check for incorrect sphinx directives' ; echo $MSG - ! grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas ./doc/source + invgrep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas ./doc/source RET=$(($RET + $?)) ; echo $MSG "DONE" MSG='Check for modules that pandas should not import' ; echo $MSG @@ -106,7 +127,7 @@ blacklist = {'bs4', 'gcsfs', 'html5lib', 'ipython', 'jinja2' 'hypothesis', 'tables', 'xlrd', 'xlsxwriter', 'xlwt'} mods = blacklist & set(m.split('.')[0] for m in sys.modules) if mods: - sys.stderr.write('pandas should not import: {}\n'.format(', '.join(mods))) + sys.stderr.write('err: pandas should not import: {}\n'.format(', '.join(mods))) sys.exit(len(mods)) " RET=$(($RET + $?)) ; echo $MSG "DONE" diff --git a/pandas/pep.py b/pandas/pep.py new file mode 100644 index 0000000000000..da147d2d0a22a --- /dev/null +++ b/pandas/pep.py @@ -0,0 +1,19 @@ +from pandas.core.common import * + + +def foo(): + return (1 + 2 + + 3 + + 4) + + + +def bar(): + """ + DEPRECATED: linting should detect this and complain because it's not + a sphinx directive + """ + foobar = 1+2+3+4 + return (1 + 2 + + 3 + + 4) From 87b5048220e42054282c517abd78842d089d02a0 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 13 Oct 2018 01:05:19 +0530 Subject: [PATCH 022/104] Trying if environment variables can be defined in the script setting --- azure-pipelines.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fda67034a43df..6af487ef09022 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -39,13 +39,11 @@ jobs: displayName: 'Installing cpplint' condition: true - # TODO update the script so flake8 uses: - # --format="##vso[task.logissue type=error;sourcepath=%(path)s;linenumber=%(row)s;columnnumber=%(col)s;code=%(code)s;]%(text)s" - - script: ci/code_checks.sh lint + - script: AZURE=true ci/code_checks.sh lint displayName: 'Linting' condition: true - - script: ci/code_checks.sh patterns + - script: AZURE=true ci/code_checks.sh patterns displayName: 'Looking for unwanted patterns' condition: true From 5ab03ab345adc52d02a17e91e25f8c204133972d Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 4 Nov 2018 08:24:25 +0000 Subject: [PATCH 023/104] Removing file with errors to be displayed, and comment in the setup.cfg --- pandas/pep.py | 19 ------------------- setup.cfg | 1 - 2 files changed, 20 deletions(-) delete mode 100644 pandas/pep.py diff --git a/pandas/pep.py b/pandas/pep.py deleted file mode 100644 index da147d2d0a22a..0000000000000 --- a/pandas/pep.py +++ /dev/null @@ -1,19 +0,0 @@ -from pandas.core.common import * - - -def foo(): - return (1 + 2 - + 3 - + 4) - - - -def bar(): - """ - DEPRECATED: linting should detect this and complain because it's not - a sphinx directive - """ - foobar = 1+2+3+4 - return (1 + 2 + - 3 + - 4) diff --git a/setup.cfg b/setup.cfg index 92157ee8792c7..e0f1ea6fe3ac9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,6 @@ exclude = doc/temp/*.py, .eggs/*.py, versioneer.py -#format = ##vso[task.logissue type=error;sourcepath=%(path)s;linenumber=%(row)s;columnnumber=%(col)s;code=%(code)s;]%(text)s [yapf] based_on_style = pep8 From ca9e12ab52666286fe0d65db2b6afa8fa1816628 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 4 Nov 2018 08:55:33 +0000 Subject: [PATCH 024/104] Changing cython cast grep to invgrep --- ci/code_checks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index b934d5e188686..a98e5aa3f1c81 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -70,7 +70,7 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then # Note: this grep pattern is (intended to be) equivalent to the python # regex r'(?])> ' MSG='Linting .pyx code for spacing conventions in casting' ; echo $MSG - ! grep -r -E --include '*.pyx' --include '*.pxi.in' '> ' pandas/_libs | grep -v '[ ->]> ' + invgrep -r -E --include '*.pyx' --include '*.pxi.in' '[\w*]> ' pandas/_libs # TODO this will conflict with #23486 and we'll wait for it to be merged, so we know the exact pattern to use RET=$(($RET + $?)) ; echo $MSG "DONE" # readability/casting: Warnings about C casting instead of C++ casting From c671da520e922dc33de97b3be2fb8b0915323a42 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 9 Nov 2018 19:05:23 +0000 Subject: [PATCH 025/104] Building source of the checks build and moving the documentation build there too --- .travis.yml | 2 +- azure-pipelines.yml | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index bf1a7b3c83e91..9357dbc126ab6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,7 +105,7 @@ before_script: script: - echo "script start" - - ci/run_build_docs.sh + #- ci/run_build_docs.sh - ci/script_single.sh - ci/script_multi.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2b38936292997..f6918abfed6c5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,7 +24,7 @@ jobs: name: WindowsPy27 vmImage: vs2017-win2016 -- job: 'Code_checks' +- job: 'Checks_and_doc' pool: vmImage: ubuntu-16.04 steps: @@ -32,11 +32,16 @@ jobs: inputs: versionSpec: '3.7' architecture: 'x64' - - script: python -m pip install flake8 - displayName: 'Installing flake8' - - script: python -m pip install cpplint - displayName: 'Installing cpplint' + - script: | + sudo apt-get install -y libc6-dev-i386 + ci/incremental/install_miniconda.sh + PATH=$HOME/miniconda3/bin:$PATH ci/incremental/setup_conda_environment.sh + displayName: 'Set up environment' + + - script: | + PATH=$HOME/miniconda3/bin:$PATH ci/incremental/build.sh + displayName: 'Build' condition: true - script: AZURE=true ci/code_checks.sh lint @@ -50,3 +55,7 @@ jobs: - script: ci/code_checks.sh doctests displayName: 'Running doctests' condition: true + + - script: DOC=true ci/run_build_docs.sh + displayName: 'Building docs' + condition: true From a6eed3e974bb141839ebc0dca4f84dcba3e92028 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 9 Nov 2018 19:15:09 +0000 Subject: [PATCH 026/104] Adding dependencies file for the checks and doc build --- azure-pipelines.yml | 4 +++- ci/deps/azure-checks-and-doc.yaml | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 ci/deps/azure-checks-and-doc.yaml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f6918abfed6c5..9b82ad3349aac 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -36,7 +36,9 @@ jobs: - script: | sudo apt-get install -y libc6-dev-i386 ci/incremental/install_miniconda.sh - PATH=$HOME/miniconda3/bin:$PATH ci/incremental/setup_conda_environment.sh + export PATH=$HOME/miniconda3/bin:$PATH + export ENV_FILE=ci/deps/azure-checks-and-doc.yaml + ci/incremental/setup_conda_environment.sh displayName: 'Set up environment' - script: | diff --git a/ci/deps/azure-checks-and-doc.yaml b/ci/deps/azure-checks-and-doc.yaml new file mode 100644 index 0000000000000..cf98e8e87a888 --- /dev/null +++ b/ci/deps/azure-checks-and-doc.yaml @@ -0,0 +1,20 @@ +name: pandas +channels: + - defaults + - conda-forge +dependencies: + - cython>=0.28.2 + - flake8>=3.5 + - flake8-comprehensions + - flake8-rst + - ipython + - matplotlib + - numpy + - pytest + - pytest-xdist + - pytest-cov + - python-dateutil + - pytz + - sphinx + - pip: + - cpplint From 3b853f9be3756ee826ecbd14ac537a13fdfc05d4 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 9 Nov 2018 19:59:26 +0000 Subject: [PATCH 027/104] Renaming conda yaml env variable --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9b82ad3349aac..670507d6fed81 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,7 +37,7 @@ jobs: sudo apt-get install -y libc6-dev-i386 ci/incremental/install_miniconda.sh export PATH=$HOME/miniconda3/bin:$PATH - export ENV_FILE=ci/deps/azure-checks-and-doc.yaml + export CONDA_ENV=ci/deps/azure-checks-and-doc.yaml ci/incremental/setup_conda_environment.sh displayName: 'Set up environment' From 850202db41f228c686d40ea449d28baff04b6cc2 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 9 Nov 2018 20:33:32 +0000 Subject: [PATCH 028/104] Fixing env variables to set up conda --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 670507d6fed81..568e0b796de4b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,7 +37,8 @@ jobs: sudo apt-get install -y libc6-dev-i386 ci/incremental/install_miniconda.sh export PATH=$HOME/miniconda3/bin:$PATH - export CONDA_ENV=ci/deps/azure-checks-and-doc.yaml + export CONDA_ENV=pandas + export ENV_FILE=ci/deps/azure-checks-and-doc.yaml ci/incremental/setup_conda_environment.sh displayName: 'Set up environment' From a896532b87ff79c221d5eef50d65d10f48a434c8 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 9 Nov 2018 20:47:48 +0000 Subject: [PATCH 029/104] Debugging why azure is not installing desired dependencies --- ci/deps/azure-checks-and-doc.yaml | 4 ++-- ci/incremental/setup_conda_environment.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/deps/azure-checks-and-doc.yaml b/ci/deps/azure-checks-and-doc.yaml index cf98e8e87a888..901a7f363b6aa 100644 --- a/ci/deps/azure-checks-and-doc.yaml +++ b/ci/deps/azure-checks-and-doc.yaml @@ -16,5 +16,5 @@ dependencies: - python-dateutil - pytz - sphinx - - pip: - - cpplint +# - pip: +# - cpplint diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index f3ac99d5e7c5a..47e4d36ea2941 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -16,6 +16,7 @@ conda remove --all -q -y -n $CONDA_ENV echo echo "[create env]" +echo $ENV_FILE time conda env create -q -n "${CONDA_ENV}" --file="${ENV_FILE}" || exit 1 # Activate first From bed55be72b8dce82b7225d41d862dc7b9b1bccc0 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 9 Nov 2018 21:57:14 +0000 Subject: [PATCH 030/104] Fixing once more env variables --- azure-pipelines.yml | 4 +++- ci/deps/azure-checks-and-doc.yaml | 4 ++-- ci/incremental/setup_conda_environment.sh | 1 - 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 568e0b796de4b..9f9bd764e8396 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -43,7 +43,9 @@ jobs: displayName: 'Set up environment' - script: | - PATH=$HOME/miniconda3/bin:$PATH ci/incremental/build.sh + export PATH=$HOME/miniconda3/bin:$PATH + export CONDA_ENV=pandas + ci/incremental/build.sh displayName: 'Build' condition: true diff --git a/ci/deps/azure-checks-and-doc.yaml b/ci/deps/azure-checks-and-doc.yaml index 901a7f363b6aa..bd0ac28b3e7bd 100644 --- a/ci/deps/azure-checks-and-doc.yaml +++ b/ci/deps/azure-checks-and-doc.yaml @@ -16,5 +16,5 @@ dependencies: - python-dateutil - pytz - sphinx -# - pip: -# - cpplint + - pip: + - cpplint diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index 47e4d36ea2941..f3ac99d5e7c5a 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -16,7 +16,6 @@ conda remove --all -q -y -n $CONDA_ENV echo echo "[create env]" -echo $ENV_FILE time conda env create -q -n "${CONDA_ENV}" --file="${ENV_FILE}" || exit 1 # Activate first From e555ce0073f5c5ed380561f05d5cedcdd99758cc Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 9 Nov 2018 22:42:27 +0000 Subject: [PATCH 031/104] Fixing indentation in dependencies file --- ci/deps/azure-checks-and-doc.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/deps/azure-checks-and-doc.yaml b/ci/deps/azure-checks-and-doc.yaml index bd0ac28b3e7bd..cf98e8e87a888 100644 --- a/ci/deps/azure-checks-and-doc.yaml +++ b/ci/deps/azure-checks-and-doc.yaml @@ -16,5 +16,5 @@ dependencies: - python-dateutil - pytz - sphinx - - pip: - - cpplint + - pip: + - cpplint From 101f7f3f179a97ccd8b9dd637cfe8f63d02e2d7e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 10 Nov 2018 00:05:00 +0000 Subject: [PATCH 032/104] Adding missing env variables and dependencies --- azure-pipelines.yml | 20 +++++++++--- ci/deps/azure-checks-and-doc.yaml | 53 ++++++++++++++++++++++++++----- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9f9bd764e8396..c146425b871dd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -49,18 +49,30 @@ jobs: displayName: 'Build' condition: true - - script: AZURE=true ci/code_checks.sh lint + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + export AZURE=true + ci/code_checks.sh lint displayName: 'Linting' condition: true - - script: AZURE=true ci/code_checks.sh patterns + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + export AZURE=true + ci/code_checks.sh patterns displayName: 'Looking for unwanted patterns' condition: true - - script: ci/code_checks.sh doctests + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/code_checks.sh doctests displayName: 'Running doctests' condition: true - - script: DOC=true ci/run_build_docs.sh + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + TRAVIS_BUILD_DIR=$HOME + DOC=true + ci/run_build_docs.sh displayName: 'Building docs' condition: true diff --git a/ci/deps/azure-checks-and-doc.yaml b/ci/deps/azure-checks-and-doc.yaml index cf98e8e87a888..302d1b76dac30 100644 --- a/ci/deps/azure-checks-and-doc.yaml +++ b/ci/deps/azure-checks-and-doc.yaml @@ -1,20 +1,57 @@ name: pandas + channels: - defaults - conda-forge + dependencies: + # required - cython>=0.28.2 - - flake8>=3.5 - - flake8-comprehensions - - flake8-rst - - ipython - - matplotlib - numpy - - pytest - - pytest-xdist - - pytest-cov + - python=3.7* - python-dateutil - pytz + + # docs + - beautifulsoup4 + - bottleneck + - fastparquet + - html5lib + - hypothesis>=3.58.0 + - ipykernel + - ipython==6.5.0 + - ipywidgets + - lxml + - matplotlib + - nbconvert + - nbformat + - nbsphinx + - notebook + - numexpr + - openpyxl + - pandoc + - pyarrow + - pyqt + - pytables + - python-snappy + - r + - rpy2 + - scipy + - seaborn - sphinx + - sqlalchemy + - statsmodels + - tzlocal + - xarray + - xlrd + - xlsxwriter + - xlwt + + # checks + - flake8>=3.5 + - flake8-comprehensions + - flake8-rst + - isort + - pytest - pip: - cpplint From fce22e6cca0d51aa03c95c8b01b340f964195c40 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 10 Nov 2018 01:00:44 +0000 Subject: [PATCH 033/104] Fixing unset env variables for the doc build --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c146425b871dd..00bde945ea542 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -71,8 +71,8 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH - TRAVIS_BUILD_DIR=$HOME - DOC=true + export TRAVIS_BUILD_DIR=$HOME + export DOC=true ci/run_build_docs.sh displayName: 'Building docs' condition: true From 167f6dc053431a5316849d4a874a312d3343489f Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 10 Nov 2018 01:21:56 +0000 Subject: [PATCH 034/104] Fixing directory for the doc build script --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 00bde945ea542..22125f34b7378 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -71,7 +71,7 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH - export TRAVIS_BUILD_DIR=$HOME + export TRAVIS_BUILD_DIR=`pwd` export DOC=true ci/run_build_docs.sh displayName: 'Building docs' From d44f189e53cbe5a1728fbc83c184f15db3c9b25e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 10 Nov 2018 11:57:58 +0000 Subject: [PATCH 035/104] WIP: Testing a simpler way to build and upload the documentation --- azure-pipelines.yml | 22 ++++++++++++++--- ci/build_docs.sh | 56 -------------------------------------------- ci/run_build_docs.sh | 10 -------- 3 files changed, 19 insertions(+), 69 deletions(-) delete mode 100755 ci/build_docs.sh delete mode 100755 ci/run_build_docs.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 22125f34b7378..bea9eacaad646 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -71,8 +71,24 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH - export TRAVIS_BUILD_DIR=`pwd` - export DOC=true - ci/run_build_docs.sh + source activate pandas + doc/make.py html displayName: 'Building docs' condition: true + + - script: | + export REPO_DIR="/tmp/pandas-docs-travis" + git config --global user.email "pandas-docs-bot@localhost" + git config --global user.name "pandas-docs-bot" + # TODO PANDAS_GH_TOKEN needs to be added as a secret variable + git clone "https://${PANDAS_GH_TOKEN}@github.com/pandas-dev/pandas-docs-travis.git" $REPO_DIR + git branch gh-pages --track origin/gh-pages + # TODO if this is the build from master, and not from a PR, the build should + # be in the root replacing the old build, and not in a new directory + cp -r doc/build/html $REPO_DIR/$(System.PullRequest.PullRequestId) + cp LICENSE $REPO_DIR/$(System.PullRequest.PullRequestId) + git add --all . + git commit -m "Added pandas documentation of pull request $(System.PullRequest.PullRequestId)" + git push + displayName: 'Uploading docs' + condition: true diff --git a/ci/build_docs.sh b/ci/build_docs.sh deleted file mode 100755 index f445447e3565c..0000000000000 --- a/ci/build_docs.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -if [ "${TRAVIS_OS_NAME}" != "linux" ]; then - echo "not doing build_docs on non-linux" - exit 0 -fi - -cd "$TRAVIS_BUILD_DIR" -echo "inside $0" - -if [ "$DOC" ]; then - - echo "Will build docs" - - source activate pandas - - mv "$TRAVIS_BUILD_DIR"/doc /tmp - mv "$TRAVIS_BUILD_DIR/LICENSE" /tmp # included in the docs. - cd /tmp/doc - - echo ############################### - echo # Log file for the doc build # - echo ############################### - - echo ./make.py - ./make.py - - echo ######################## - echo # Create and send docs # - echo ######################## - - cd /tmp/doc/build/html - git config --global user.email "pandas-docs-bot@localhost.foo" - git config --global user.name "pandas-docs-bot" - - # create the repo - git init - - touch README - git add README - git commit -m "Initial commit" --allow-empty - git branch gh-pages - git checkout gh-pages - touch .nojekyll - git add --all . - git commit -m "Version" --allow-empty - - git remote remove origin - git remote add origin "https://${PANDAS_GH_TOKEN}@github.com/pandas-dev/pandas-docs-travis.git" - git fetch origin - git remote -v - - git push origin gh-pages -f -fi - -exit 0 diff --git a/ci/run_build_docs.sh b/ci/run_build_docs.sh deleted file mode 100755 index 2909b9619552e..0000000000000 --- a/ci/run_build_docs.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -echo "inside $0" - -"$TRAVIS_BUILD_DIR"/ci/build_docs.sh 2>&1 - -# wait until subprocesses finish (build_docs.sh) -wait - -exit 0 From 906da221b675c2a686bd506bc9515bf0a57a4c74 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 10 Nov 2018 23:04:48 +0000 Subject: [PATCH 036/104] Printing installed dependencies in checks job (temporary, to debug pyperclip error), and adding docstring validation to the job --- azure-pipelines.yml | 7 +++++++ ci/code_checks.sh | 17 ++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bea9eacaad646..b7c0f0f750061 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -40,6 +40,7 @@ jobs: export CONDA_ENV=pandas export ENV_FILE=ci/deps/azure-checks-and-doc.yaml ci/incremental/setup_conda_environment.sh + conda env list displayName: 'Set up environment' - script: | @@ -69,6 +70,12 @@ jobs: displayName: 'Running doctests' condition: true + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/code_checks.sh docstrings + displayName: 'Docstring validation' + condition: true + - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas diff --git a/ci/code_checks.sh b/ci/code_checks.sh index efddc1f6bd774..a4aa25d06aea5 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -5,16 +5,18 @@ # This script is intended for both the CI and to check locally that code standards are # respected. We are currently linting (PEP-8 and similar), looking for patterns of # common mistakes (sphinx directives with missing blank lines, old style classes, -# unwanted imports...), and we also run doctests here (currently some files only). -# In the future we may want to add the validation of docstrings and other checks here. +# unwanted imports...), we run doctests here (currently some files only), and we +# validate formatting error in docstrings. # # Usage: # $ ./ci/code_checks.sh # run all checks # $ ./ci/code_checks.sh lint # run linting only # $ ./ci/code_checks.sh patterns # check for patterns that should not exist # $ ./ci/code_checks.sh doctests # run doctests +# $ ./ci/code_checks.sh docstrings # validate docstring errors -[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "doctests" ]] || { echo "Unknown command $1. Usage: $0 [lint|patterns|doctests]"; exit 9999; } +[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "doctests" || "$1" == "docstrings" ]] || \ + { echo "Unknown command $1. Usage: $0 [lint|patterns|doctests|docstrings]"; exit 9999; } source activate pandas RET=0 @@ -193,4 +195,13 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then fi +### DOCSTRINGS ### +if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then + + MSG='Validate docstrings (SS04, EX04)' ; echo $MSG + scripts/validate_docstrings.py --format=azure --errors=SS04,EX04 + RET=$(($RET + $?)) ; echo $MSG "DONE" + +fi + exit $RET From 9fb399945cf935a7c6fc9a9930f3e4caa8e42c8f Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 11 Nov 2018 00:59:46 +0000 Subject: [PATCH 037/104] Printing installed dependencies in azure, for debugging --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b7c0f0f750061..6802455009c4f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -40,7 +40,7 @@ jobs: export CONDA_ENV=pandas export ENV_FILE=ci/deps/azure-checks-and-doc.yaml ci/incremental/setup_conda_environment.sh - conda env list + conda list -e # TODO just for debugging, remove later displayName: 'Set up environment' - script: | From c72c0e53808becbdafd4074ac38d12343b52e39b Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 11 Nov 2018 01:02:59 +0000 Subject: [PATCH 038/104] Fixing bug when uploading the docs to github pages --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6802455009c4f..1eeb720832207 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -90,6 +90,7 @@ jobs: # TODO PANDAS_GH_TOKEN needs to be added as a secret variable git clone "https://${PANDAS_GH_TOKEN}@github.com/pandas-dev/pandas-docs-travis.git" $REPO_DIR git branch gh-pages --track origin/gh-pages + git checkout gh-pages # TODO if this is the build from master, and not from a PR, the build should # be in the root replacing the old build, and not in a new directory cp -r doc/build/html $REPO_DIR/$(System.PullRequest.PullRequestId) From de1f52ff6809e2757a8672aa5d6e647b3c482b79 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 11 Nov 2018 01:08:22 +0000 Subject: [PATCH 039/104] Adding an extra error to the validation of docstrings, as it's been fixed everywhere --- ci/code_checks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index a4aa25d06aea5..833f2c9b2e4f4 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -199,7 +199,7 @@ fi if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then MSG='Validate docstrings (SS04, EX04)' ; echo $MSG - scripts/validate_docstrings.py --format=azure --errors=SS04,EX04 + scripts/validate_docstrings.py --format=azure --errors=SS04,EX04,PR05 RET=$(($RET + $?)) ; echo $MSG "DONE" fi From e4aa3715aece703a85b0bebdb2766d3d4c717472 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 11 Nov 2018 17:57:20 +0000 Subject: [PATCH 040/104] Adding new docstring error code to the validation, and tests with the pyqt error --- azure-pipelines.yml | 1 + ci/code_checks.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1eeb720832207..4bc119fc0472b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -66,6 +66,7 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH + conda install pyqt # TODO should be in the dependencies, but not sure if it's being installed, just testing what happens ci/code_checks.sh doctests displayName: 'Running doctests' condition: true diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 833f2c9b2e4f4..1628b80a56440 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -198,8 +198,8 @@ fi ### DOCSTRINGS ### if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then - MSG='Validate docstrings (SS04, EX04)' ; echo $MSG - scripts/validate_docstrings.py --format=azure --errors=SS04,EX04,PR05 + MSG='Validate docstrings (SS04, PR03, PR05, EX04)' ; echo $MSG + scripts/validate_docstrings.py --format=azure --errors=SS04,PR03,PR05,EX04 RET=$(($RET + $?)) ; echo $MSG "DONE" fi From 63cfd5b47654a596a9a05f05e11a8a7bb66c9148 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 12 Nov 2018 16:56:59 +0000 Subject: [PATCH 041/104] Generating the script of the deps check in azure format --- scripts/generate_pip_deps_from_conda.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/generate_pip_deps_from_conda.py b/scripts/generate_pip_deps_from_conda.py index 2474214a4a53b..a1b5bd2c3e150 100755 --- a/scripts/generate_pip_deps_from_conda.py +++ b/scripts/generate_pip_deps_from_conda.py @@ -83,6 +83,9 @@ def main(conda_fname, pip_fname, compare=False): argparser.add_argument('--compare', action='store_true', help='compare whether the two files are equivalent') + argparser.add_argument('--azure', + action='store_true', + help='show the output in azure-pipelines format') args = argparser.parse_args() repo_path = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) @@ -90,7 +93,10 @@ def main(conda_fname, pip_fname, compare=False): os.path.join(repo_path, 'requirements-dev.txt'), compare=args.compare) if res: - sys.stderr.write('`requirements-dev.txt` has to be generated with ' - '`{}` after `environment.yml` is modified.\n'.format( - sys.argv[0])) + msg = ('`requirements-dev.txt` has to be generated with `{}` after ' + '`environment.yml` is modified.\n'.format(sys.argv[0])) + if args.azure: + msg = ('##vso[task.logissue type=error;' + 'sourcepath=requirements-dev.txt]{}'.format(msg)) + sys.stderr.write(msg) sys.exit(res) From eb558dfd6575d038553a77acec79f73c1a4f7c25 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 12 Nov 2018 16:57:47 +0000 Subject: [PATCH 042/104] Publishing docs with azure artifacts instead of github pages, and adding better debug info on installed deps --- azure-pipelines.yml | 26 +++++++---------------- ci/code_checks.sh | 4 ++-- ci/incremental/setup_conda_environment.sh | 3 +++ 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4bc119fc0472b..cd2ccb34b4fca 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -40,7 +40,6 @@ jobs: export CONDA_ENV=pandas export ENV_FILE=ci/deps/azure-checks-and-doc.yaml ci/incremental/setup_conda_environment.sh - conda list -e # TODO just for debugging, remove later displayName: 'Set up environment' - script: | @@ -66,7 +65,6 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH - conda install pyqt # TODO should be in the dependencies, but not sure if it's being installed, just testing what happens ci/code_checks.sh doctests displayName: 'Running doctests' condition: true @@ -84,20 +82,12 @@ jobs: displayName: 'Building docs' condition: true - - script: | - export REPO_DIR="/tmp/pandas-docs-travis" - git config --global user.email "pandas-docs-bot@localhost" - git config --global user.name "pandas-docs-bot" - # TODO PANDAS_GH_TOKEN needs to be added as a secret variable - git clone "https://${PANDAS_GH_TOKEN}@github.com/pandas-dev/pandas-docs-travis.git" $REPO_DIR - git branch gh-pages --track origin/gh-pages - git checkout gh-pages - # TODO if this is the build from master, and not from a PR, the build should - # be in the root replacing the old build, and not in a new directory - cp -r doc/build/html $REPO_DIR/$(System.PullRequest.PullRequestId) - cp LICENSE $REPO_DIR/$(System.PullRequest.PullRequestId) - git add --all . - git commit -m "Added pandas documentation of pull request $(System.PullRequest.PullRequestId)" - git push - displayName: 'Uploading docs' + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: $(Build.SourcesDirectory)/doc/build/html + artifactName: 'docs' + publishLocation: 'Container' + parallel: true + parallelCount: 8 + displayName: 'Publshing docs' condition: true diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 5f183dd1affef..22f4f411b1b53 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -205,7 +205,7 @@ fi if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then MSG='Validate docstrings (SS04, PR03, PR05, EX04)' ; echo $MSG - scripts/validate_docstrings.py --format=azure --errors=SS04,PR03,PR05,EX04 + $BASE_DIR/scripts/validate_docstrings.py --format=azure --errors=SS04,PR03,PR05,EX04 RET=$(($RET + $?)) ; echo $MSG "DONE" fi @@ -214,7 +214,7 @@ fi if [[ -z "$CHECK" || "$CHECK" == "dependencies" ]]; then MSG='Check that requirements-dev.txt has been generated from environment.yml' ; echo $MSG - $BASE_DIR/scripts/generate_pip_deps_from_conda.py --compare + $BASE_DIR/scripts/generate_pip_deps_from_conda.py --compare --azure RET=$(($RET + $?)) ; echo $MSG "DONE" fi diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index f3ac99d5e7c5a..68f00d820dd4b 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -17,6 +17,7 @@ conda remove --all -q -y -n $CONDA_ENV echo echo "[create env]" time conda env create -q -n "${CONDA_ENV}" --file="${ENV_FILE}" || exit 1 +conda list -e # Activate first set +v @@ -38,6 +39,8 @@ if [ -n "$LOCALE_OVERRIDE" ]; then sudo locale-gen "$LOCALE_OVERRIDE" fi +conda list -e + # # Install the compiler toolchain # if [[ $(uname) == Linux ]]; then # if [[ "$CONDA_SUBDIR" == "linux-32" || "$BITS32" == "yes" ]] ; then From 371f06e7645e3506e01a5b85e481f03633354fb7 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 13 Nov 2018 00:35:49 +0000 Subject: [PATCH 043/104] Removing unnecessary checks on travis on whether the doc is being built, as it is built in azure now --- ci/script_multi.sh | 5 +---- ci/script_single.sh | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/ci/script_multi.sh b/ci/script_multi.sh index e56d5da7232b2..2fbda3cdd2792 100755 --- a/ci/script_multi.sh +++ b/ci/script_multi.sh @@ -23,10 +23,7 @@ fi export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))') echo PYTHONHASHSEED=$PYTHONHASHSEED -if [ "$DOC" ]; then - echo "We are not running pytest as this is a doc-build" - -elif [ "$COVERAGE" ]; then +if [ "$COVERAGE" ]; then echo pytest -s -n 2 -m "not single" --durations=10 --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas pytest -s -n 2 -m "not single" --durations=10 --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas diff --git a/ci/script_single.sh b/ci/script_single.sh index ea0d48bc2da8a..bc34a62a7d6b4 100755 --- a/ci/script_single.sh +++ b/ci/script_single.sh @@ -22,10 +22,7 @@ if echo "$TEST_ARGS" | grep -e --skip-network -q; then export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4; fi -if [ "$DOC" ]; then - echo "We are not running pytest as this is a doc-build" - -elif [ "$COVERAGE" ]; then +if [ "$COVERAGE" ]; then echo pytest -s -m "single" --durations=10 --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=test-data-single.xml $TEST_ARGS pandas pytest -s -m "single" --durations=10 --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=test-data-single.xml $TEST_ARGS pandas echo pytest -s --strict scripts From d262a04843990bdc9e3d30de4b4ef34f9a6dcfee Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 13 Nov 2018 00:43:22 +0000 Subject: [PATCH 044/104] Debugging why pyqt is not being installed --- ci/incremental/setup_conda_environment.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index 68f00d820dd4b..57e6120c65ddb 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -16,9 +16,16 @@ conda remove --all -q -y -n $CONDA_ENV echo echo "[create env]" +echo $ENV_FILE +cat $ENV_FILE time conda env create -q -n "${CONDA_ENV}" --file="${ENV_FILE}" || exit 1 conda list -e +echo "Installing pyqt, as seems it's not got from the file (debugging why)" +conda install pyqt +conda list -e + + # Activate first set +v source activate $CONDA_ENV From 168ec55e5c29e44c17d1989ee4202df84019dce4 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 13 Nov 2018 00:53:59 +0000 Subject: [PATCH 045/104] Adding section names check to validate docstrings, now that all wrong sections have been fixed --- ci/code_checks.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 22f4f411b1b53..03edc54d35cab 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -204,8 +204,8 @@ fi ### DOCSTRINGS ### if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then - MSG='Validate docstrings (SS04, PR03, PR05, EX04)' ; echo $MSG - $BASE_DIR/scripts/validate_docstrings.py --format=azure --errors=SS04,PR03,PR05,EX04 + MSG='Validate docstrings (GL06, SS04, PR03, PR05, EX04)' ; echo $MSG + $BASE_DIR/scripts/validate_docstrings.py --format=azure --errors=GL06,SS04,PR03,PR05,EX04 RET=$(($RET + $?)) ; echo $MSG "DONE" fi From 2f1b270a5994e6db329935dc125f15c19094e64d Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 13 Nov 2018 22:14:57 +0000 Subject: [PATCH 046/104] Fixing conda install pyqt for debugging --- .travis.yml | 1 - ci/incremental/setup_conda_environment.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9357dbc126ab6..1e7e51da7262e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,7 +105,6 @@ before_script: script: - echo "script start" - #- ci/run_build_docs.sh - ci/script_single.sh - ci/script_multi.sh diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index 57e6120c65ddb..4d9bca18f87e9 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -22,7 +22,7 @@ time conda env create -q -n "${CONDA_ENV}" --file="${ENV_FILE}" || exit 1 conda list -e echo "Installing pyqt, as seems it's not got from the file (debugging why)" -conda install pyqt +conda install -yq pyqt conda list -e From a3f601c1e664d0e723e51484c038439fa0d3f75a Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 19 Nov 2018 15:57:33 +0000 Subject: [PATCH 047/104] Using github pages to publish the documentation --- azure-pipelines.yml | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cd2ccb34b4fca..446d368941d80 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -82,12 +82,26 @@ jobs: displayName: 'Building docs' condition: true - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: $(Build.SourcesDirectory)/doc/build/html - artifactName: 'docs' - publishLocation: 'Container' - parallel: true - parallelCount: 8 - displayName: 'Publshing docs' + - script: | + export REPO_DIR=$(Build.ArtifactStagingDirectory)/pandas-docs-travis + if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then + export NAME="master" + else + export NAME=$(System.PullRequest.PullRequestId) + fi + echo $(Build.SourceBranch) + echo $NAME + export PR_DIR=$REPO_DIR/pr/$NAME + git clone https://$(GITHUB_DOC_TOKEN)@github.com/pandas-dev/pandas-docs-travis.git $REPO_DIR + mkdir -p $PR_DIR + rm -rf $PR_DIR/* + cp -r doc/build/html/* $PR_DIR/ + cp LICENSE $PR_DIR/ + cd $REPO_DIR + git add --all . + git config user.email "pandas-docs-bot@localhost" + git config user.name "pandas-docs-bot" + git commit -m "Added pandas documentation for $NAME" + git push + displayName: 'Publishing docs' condition: true From 5ea21c60a28ac253b9c517d55663d0a639cdd271 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 19 Nov 2018 16:40:48 +0000 Subject: [PATCH 048/104] Testing if linux variables can be defined once for the whole job --- azure-pipelines.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 446d368941d80..4e912b693a49e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -27,6 +27,8 @@ jobs: - job: 'Checks_and_doc' pool: vmImage: ubuntu-16.04 + env: + CONDA_ENV: pandas steps: - task: UsePythonVersion@0 inputs: @@ -37,14 +39,14 @@ jobs: sudo apt-get install -y libc6-dev-i386 ci/incremental/install_miniconda.sh export PATH=$HOME/miniconda3/bin:$PATH - export CONDA_ENV=pandas + #export CONDA_ENV=pandas export ENV_FILE=ci/deps/azure-checks-and-doc.yaml ci/incremental/setup_conda_environment.sh displayName: 'Set up environment' - script: | export PATH=$HOME/miniconda3/bin:$PATH - export CONDA_ENV=pandas + #export CONDA_ENV=pandas ci/incremental/build.sh displayName: 'Build' condition: true From 46d281c61345b968c96344bdccb2fcd165c82ebd Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 19 Nov 2018 16:59:52 +0000 Subject: [PATCH 049/104] Using env for the environment variables, and implemented azure storage publishing --- azure-pipelines.yml | 64 +++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4e912b693a49e..788bcfbdceb7c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -27,8 +27,6 @@ jobs: - job: 'Checks_and_doc' pool: vmImage: ubuntu-16.04 - env: - CONDA_ENV: pandas steps: - task: UsePythonVersion@0 inputs: @@ -38,61 +36,69 @@ jobs: - script: | sudo apt-get install -y libc6-dev-i386 ci/incremental/install_miniconda.sh - export PATH=$HOME/miniconda3/bin:$PATH - #export CONDA_ENV=pandas - export ENV_FILE=ci/deps/azure-checks-and-doc.yaml ci/incremental/setup_conda_environment.sh displayName: 'Set up environment' + env: + PATH: $HOME/miniconda3/bin:$PATH + CONDA_ENV: pandas + ENV_FILE: ci/deps/azure-checks-and-doc.yaml - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - #export CONDA_ENV=pandas - ci/incremental/build.sh + - script: ci/incremental/build.sh displayName: 'Build' condition: true + env: + PATH: $HOME/miniconda3/bin:$PATH + CONDA_ENV: pandas - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - export AZURE=true - ci/code_checks.sh lint + - script: ci/code_checks.sh lint displayName: 'Linting' condition: true + env: + PATH: $HOME/miniconda3/bin:$PATH + AZURE: true - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - export AZURE=true - ci/code_checks.sh patterns + - script: ci/code_checks.sh patterns displayName: 'Looking for unwanted patterns' condition: true + env: + PATH: $HOME/miniconda3/bin:$PATH + AZURE: true - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - ci/code_checks.sh doctests + - script: ci/code_checks.sh doctests displayName: 'Running doctests' condition: true + env: + PATH: $HOME/miniconda3/bin:$PATH - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - ci/code_checks.sh docstrings + - script: ci/code_checks.sh docstrings displayName: 'Docstring validation' condition: true + env: + PATH: $HOME/miniconda3/bin:$PATH - script: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas doc/make.py html displayName: 'Building docs' condition: true + env: + PATH: $HOME/miniconda3/bin:$PATH + + - scripts: | + az extension add --name storage-preview + az storage blob upload-batch -s $SOURCE_PATH -d '$web' + displayName: 'Publishing docs (Azure storage)' + condition: true + env: + SOURCE_PATH: $(Build.SourcesDirectory)/doc/build/html/ + AZURE_STORAGE_CONNECTION_STRING: $(CONNECTION_STRING) - script: | - export REPO_DIR=$(Build.ArtifactStagingDirectory)/pandas-docs-travis if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then export NAME="master" else export NAME=$(System.PullRequest.PullRequestId) fi - echo $(Build.SourceBranch) - echo $NAME export PR_DIR=$REPO_DIR/pr/$NAME git clone https://$(GITHUB_DOC_TOKEN)@github.com/pandas-dev/pandas-docs-travis.git $REPO_DIR mkdir -p $PR_DIR @@ -105,5 +111,7 @@ jobs: git config user.name "pandas-docs-bot" git commit -m "Added pandas documentation for $NAME" git push - displayName: 'Publishing docs' + displayName: 'Publishing docs (GitHub pages)' condition: true + env: + REPO_DIR: $(Build.ArtifactStagingDirectory)/pandas-docs-travis From faf49e90bc16f0874d76696015cbe6861763ba5e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 19 Nov 2018 17:03:35 +0000 Subject: [PATCH 050/104] Fixing typo --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 788bcfbdceb7c..dd75ac23642ab 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -84,7 +84,7 @@ jobs: env: PATH: $HOME/miniconda3/bin:$PATH - - scripts: | + - script: | az extension add --name storage-preview az storage blob upload-batch -s $SOURCE_PATH -d '$web' displayName: 'Publishing docs (Azure storage)' From fa4f16cb1fa937609e4f5a26f27b510e46557345 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 19 Nov 2018 17:34:03 +0000 Subject: [PATCH 051/104] Setting environment variables at the beginning of the job, and adding debug information on directories --- azure-pipelines.yml | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index dd75ac23642ab..38be2c1295ede 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -34,55 +34,53 @@ jobs: architecture: 'x64' - script: | + echo "HOME" + echo $HOME + echo "Build.SourcesDirectory" + echo $(Build.SourcesDirectory) + echo "pwd" + pwd + displayName: "TEMPORARY checking directories" + + - script: | + echo '##vso[task.setvariable variable=PATH]$HOME/miniconda3/bin:$PATH' + echo '##vso[task.setvariable variable=CONDA_ENV]pandas' + echo '##vso[task.setvariable variable=ENV_FILE]ci/deps/azure-checks-and-doc.yaml' + echo '##vso[task.setvariable variable=AZURE]true' + displayName: 'Setting environment variables' + + - script: | + echo $PATH sudo apt-get install -y libc6-dev-i386 ci/incremental/install_miniconda.sh ci/incremental/setup_conda_environment.sh displayName: 'Set up environment' - env: - PATH: $HOME/miniconda3/bin:$PATH - CONDA_ENV: pandas - ENV_FILE: ci/deps/azure-checks-and-doc.yaml - script: ci/incremental/build.sh displayName: 'Build' condition: true - env: - PATH: $HOME/miniconda3/bin:$PATH - CONDA_ENV: pandas - script: ci/code_checks.sh lint displayName: 'Linting' condition: true - env: - PATH: $HOME/miniconda3/bin:$PATH - AZURE: true - script: ci/code_checks.sh patterns displayName: 'Looking for unwanted patterns' condition: true - env: - PATH: $HOME/miniconda3/bin:$PATH - AZURE: true - script: ci/code_checks.sh doctests displayName: 'Running doctests' condition: true - env: - PATH: $HOME/miniconda3/bin:$PATH - script: ci/code_checks.sh docstrings displayName: 'Docstring validation' condition: true - env: - PATH: $HOME/miniconda3/bin:$PATH - script: | source activate pandas doc/make.py html displayName: 'Building docs' condition: true - env: - PATH: $HOME/miniconda3/bin:$PATH - script: | az extension add --name storage-preview From c05f6b7e507f6d0c8325e7efaa4fc4d8c8416917 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 19 Nov 2018 17:49:23 +0000 Subject: [PATCH 052/104] Fixing /home/mgarcia/miniconda3/envs/pandas-dev/bin:/home/mgarcia/miniconda3/bin:/home/mgarcia/miniconda3/bin:/usr/lib64/qt-3.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/home/mgarcia/.local/bin:/home/mgarcia/bin variable --- azure-pipelines.yml | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 38be2c1295ede..aabdf7d9c23d6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -34,49 +34,52 @@ jobs: architecture: 'x64' - script: | - echo "HOME" - echo $HOME - echo "Build.SourcesDirectory" - echo $(Build.SourcesDirectory) - echo "pwd" - pwd - displayName: "TEMPORARY checking directories" - - - script: | - echo '##vso[task.setvariable variable=PATH]$HOME/miniconda3/bin:$PATH' + # XXX not sure why setting $PATH here does not work + #echo '##vso[task.setvariable variable=PATH]$HOME/miniconda3/bin:$PATH' echo '##vso[task.setvariable variable=CONDA_ENV]pandas' echo '##vso[task.setvariable variable=ENV_FILE]ci/deps/azure-checks-and-doc.yaml' echo '##vso[task.setvariable variable=AZURE]true' displayName: 'Setting environment variables' - script: | - echo $PATH + export PATH=$HOME/miniconda3/bin:$PATH' sudo apt-get install -y libc6-dev-i386 ci/incremental/install_miniconda.sh ci/incremental/setup_conda_environment.sh displayName: 'Set up environment' - - script: ci/incremental/build.sh + - script: | + export PATH=$HOME/miniconda3/bin:$PATH' + ci/incremental/build.sh displayName: 'Build' condition: true - - script: ci/code_checks.sh lint + - script: | + export PATH=$HOME/miniconda3/bin:$PATH' + ci/code_checks.sh lint displayName: 'Linting' condition: true - - script: ci/code_checks.sh patterns + - script: | + export PATH=$HOME/miniconda3/bin:$PATH' + ci/code_checks.sh patterns displayName: 'Looking for unwanted patterns' condition: true - - script: ci/code_checks.sh doctests + - script: | + export PATH=$HOME/miniconda3/bin:$PATH' + ci/code_checks.sh doctests displayName: 'Running doctests' condition: true - - script: ci/code_checks.sh docstrings + - script: | + export PATH=$HOME/miniconda3/bin:$PATH' + ci/code_checks.sh docstrings displayName: 'Docstring validation' condition: true - script: | + export PATH=$HOME/miniconda3/bin:$PATH' source activate pandas doc/make.py html displayName: 'Building docs' From 9162aeb488549c44c626b9a60be10baf118944ef Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 19 Nov 2018 17:55:23 +0000 Subject: [PATCH 053/104] Fixed typo in PATH --- azure-pipelines.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index aabdf7d9c23d6..b430665516ccd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -42,44 +42,44 @@ jobs: displayName: 'Setting environment variables' - script: | - export PATH=$HOME/miniconda3/bin:$PATH' + export PATH=$HOME/miniconda3/bin:$PATH sudo apt-get install -y libc6-dev-i386 ci/incremental/install_miniconda.sh ci/incremental/setup_conda_environment.sh displayName: 'Set up environment' - script: | - export PATH=$HOME/miniconda3/bin:$PATH' + export PATH=$HOME/miniconda3/bin:$PATH ci/incremental/build.sh displayName: 'Build' condition: true - script: | - export PATH=$HOME/miniconda3/bin:$PATH' + export PATH=$HOME/miniconda3/bin:$PATH ci/code_checks.sh lint displayName: 'Linting' condition: true - script: | - export PATH=$HOME/miniconda3/bin:$PATH' + export PATH=$HOME/miniconda3/bin:$PATH ci/code_checks.sh patterns displayName: 'Looking for unwanted patterns' condition: true - script: | - export PATH=$HOME/miniconda3/bin:$PATH' + export PATH=$HOME/miniconda3/bin:$PATH ci/code_checks.sh doctests displayName: 'Running doctests' condition: true - script: | - export PATH=$HOME/miniconda3/bin:$PATH' + export PATH=$HOME/miniconda3/bin:$PATH ci/code_checks.sh docstrings displayName: 'Docstring validation' condition: true - script: | - export PATH=$HOME/miniconda3/bin:$PATH' + export PATH=$HOME/miniconda3/bin:$PATH source activate pandas doc/make.py html displayName: 'Building docs' From 564090717c335c5c9fd0ea821a44fa76431848bf Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 19 Nov 2018 19:57:42 +0000 Subject: [PATCH 054/104] Formatting flake8-rst output for azure --- ci/code_checks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 03edc54d35cab..1f52e93f977d3 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -73,7 +73,7 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then flake8-rst --version MSG='Linting code-blocks in .rst documentation' ; echo $MSG - flake8-rst doc/source --filename=*.rst + flake8-rst doc/source --filename=*.rst --format="$FLAKE8_FORMAT" RET=$(($RET + $?)) ; echo $MSG "DONE" # Check that cython casting is of the form `obj` as opposed to ` obj`; From 629a2098d7ee07047e3d31b5af8a20d7c9f33178 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 20 Nov 2018 17:42:40 +0000 Subject: [PATCH 055/104] Updating azure connection string --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b430665516ccd..5ea0f847223be 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -92,7 +92,7 @@ jobs: condition: true env: SOURCE_PATH: $(Build.SourcesDirectory)/doc/build/html/ - AZURE_STORAGE_CONNECTION_STRING: $(CONNECTION_STRING) + AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=https;AccountName=$(AZURE_STORAGE_ACCOUNT);AccountKey=$(AZURE_STORAGE_ACCESS_KEY) - script: | if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then From 588d153c9217be620db54322bec2a4b1a6954f60 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 20 Nov 2018 17:48:25 +0000 Subject: [PATCH 056/104] Checking if environment.yml can be used for the docs and checks build, and adding debug information for the doctests error --- azure-pipelines.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5ea0f847223be..7c5266c0b41ab 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -35,9 +35,12 @@ jobs: - script: | # XXX not sure why setting $PATH here does not work - #echo '##vso[task.setvariable variable=PATH]$HOME/miniconda3/bin:$PATH' + # echo '##vso[task.setvariable variable=PATH]$HOME/miniconda3/bin:$PATH' echo '##vso[task.setvariable variable=CONDA_ENV]pandas' - echo '##vso[task.setvariable variable=ENV_FILE]ci/deps/azure-checks-and-doc.yaml' + # TODO remove the deps file if `environment.yml` can be used + # echo '##vso[task.setvariable variable=ENV_FILE]ci/deps/azure-checks-and-doc.yaml' + # XXX this build should run with the same environment we use locally + echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' echo '##vso[task.setvariable variable=AZURE]true' displayName: 'Setting environment variables' @@ -68,6 +71,7 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH + conda list | grep qt # XXX only for debugging during implementation ci/code_checks.sh doctests displayName: 'Running doctests' condition: true From bc9c3b3e2949494766d7b585f6a2bdaf660a3638 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 20 Nov 2018 18:25:54 +0000 Subject: [PATCH 057/104] Remove unused stuff from azure config --- azure-pipelines.yml | 6 +--- ci/deps/azure-checks-and-doc.yaml | 57 ------------------------------- 2 files changed, 1 insertion(+), 62 deletions(-) delete mode 100644 ci/deps/azure-checks-and-doc.yaml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7c5266c0b41ab..0c9161e6ee246 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,9 +37,6 @@ jobs: # XXX not sure why setting $PATH here does not work # echo '##vso[task.setvariable variable=PATH]$HOME/miniconda3/bin:$PATH' echo '##vso[task.setvariable variable=CONDA_ENV]pandas' - # TODO remove the deps file if `environment.yml` can be used - # echo '##vso[task.setvariable variable=ENV_FILE]ci/deps/azure-checks-and-doc.yaml' - # XXX this build should run with the same environment we use locally echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' echo '##vso[task.setvariable variable=AZURE]true' displayName: 'Setting environment variables' @@ -71,7 +68,6 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH - conda list | grep qt # XXX only for debugging during implementation ci/code_checks.sh doctests displayName: 'Running doctests' condition: true @@ -96,7 +92,7 @@ jobs: condition: true env: SOURCE_PATH: $(Build.SourcesDirectory)/doc/build/html/ - AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=https;AccountName=$(AZURE_STORAGE_ACCOUNT);AccountKey=$(AZURE_STORAGE_ACCESS_KEY) + AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=https;AccountName=$(AZURE_STORAGE_ACCOUNT);AccountKey=$(AZURE_STORAGE_ACCESS_KEY)" - script: | if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then diff --git a/ci/deps/azure-checks-and-doc.yaml b/ci/deps/azure-checks-and-doc.yaml deleted file mode 100644 index 302d1b76dac30..0000000000000 --- a/ci/deps/azure-checks-and-doc.yaml +++ /dev/null @@ -1,57 +0,0 @@ -name: pandas - -channels: - - defaults - - conda-forge - -dependencies: - # required - - cython>=0.28.2 - - numpy - - python=3.7* - - python-dateutil - - pytz - - # docs - - beautifulsoup4 - - bottleneck - - fastparquet - - html5lib - - hypothesis>=3.58.0 - - ipykernel - - ipython==6.5.0 - - ipywidgets - - lxml - - matplotlib - - nbconvert - - nbformat - - nbsphinx - - notebook - - numexpr - - openpyxl - - pandoc - - pyarrow - - pyqt - - pytables - - python-snappy - - r - - rpy2 - - scipy - - seaborn - - sphinx - - sqlalchemy - - statsmodels - - tzlocal - - xarray - - xlrd - - xlsxwriter - - xlwt - - # checks - - flake8>=3.5 - - flake8-comprehensions - - flake8-rst - - isort - - pytest - - pip: - - cpplint From 324b1e2ca0c8052b25872b15428137956348f05d Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 20 Nov 2018 18:30:46 +0000 Subject: [PATCH 058/104] Doing checks as soon as possible (before conda, before build) --- azure-pipelines.yml | 27 +++++++++++++++++++++------ ci/code_checks.sh | 20 +++++++++++++------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0c9161e6ee246..02b056ec63557 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -41,6 +41,13 @@ jobs: echo '##vso[task.setvariable variable=AZURE]true' displayName: 'Setting environment variables' + # Does not require a conda environment + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/code_checks.sh patterns + displayName: 'Looking for unwanted patterns' + condition: true + - script: | export PATH=$HOME/miniconda3/bin:$PATH sudo apt-get install -y libc6-dev-i386 @@ -48,24 +55,32 @@ jobs: ci/incremental/setup_conda_environment.sh displayName: 'Set up environment' + # Does not require pandas - script: | export PATH=$HOME/miniconda3/bin:$PATH - ci/incremental/build.sh - displayName: 'Build' + ci/code_checks.sh lint + displayName: 'Linting' condition: true - script: | export PATH=$HOME/miniconda3/bin:$PATH - ci/code_checks.sh lint - displayName: 'Linting' + ci/code_checks.sh code + displayName: 'Checks on imported code' condition: true - script: | export PATH=$HOME/miniconda3/bin:$PATH - ci/code_checks.sh patterns - displayName: 'Looking for unwanted patterns' + ci/code_checks.sh dependencies + displayName: 'Dependencies consistency' + condition: true + + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/incremental/build.sh + displayName: 'Build' condition: true + # Requires pandas - script: | export PATH=$HOME/miniconda3/bin:$PATH ci/code_checks.sh doctests diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 1f52e93f977d3..bb3500009f0c3 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -9,15 +9,16 @@ # validate formatting error in docstrings. # # Usage: -# $ ./ci/code_checks.sh # run all checks -# $ ./ci/code_checks.sh lint # run linting only -# $ ./ci/code_checks.sh patterns # check for patterns that should not exist -# $ ./ci/code_checks.sh doctests # run doctests -# $ ./ci/code_checks.sh docstrings # validate docstring errors +# $ ./ci/code_checks.sh # run all checks +# $ ./ci/code_checks.sh lint # run linting only +# $ ./ci/code_checks.sh patterns # check for patterns that should not exist +# $ ./ci/code_checks.sh code # checks on imported code +# $ ./ci/code_checks.sh doctests # run doctests +# $ ./ci/code_checks.sh docstrings # validate docstring errors # $ ./ci/code_checks.sh dependencies # check that dependencies are consistent -[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "dependecies" ]] || \ - { echo "Unknown command $1. Usage: $0 [lint|patterns|doctests|docstrings|dependencies]"; exit 9999; } +[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "dependecies" ]] || \ + { echo "Unknown command $1. Usage: $0 [lint|patterns|code|doctests|docstrings|dependencies]"; exit 9999; } source activate pandas BASE_DIR="$(dirname $0)/.." @@ -148,6 +149,11 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then ! grep -R --exclude=*.pyc --exclude=testing.py --exclude=test_testing.py assert_raises_regex pandas RET=$(($RET + $?)) ; echo $MSG "DONE" +fi + +### CODE ### +if [[ -z "$CHECK" || "$CHECK" == "code" ]]; then + MSG='Check for modules that pandas should not import' ; echo $MSG python -c " import sys From 5bf17093d0a79aae26df3a36dc91b4c0b6a6f089 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 20 Nov 2018 19:50:13 +0000 Subject: [PATCH 059/104] Fixing build errors (typo in dependencies, missing cpplint, code checks before build), and clean up of travis dependencies --- azure-pipelines.yml | 12 ++++++------ ci/code_checks.sh | 2 +- ci/deps/travis-36.yaml | 11 ----------- environment.yml | 2 ++ 4 files changed, 9 insertions(+), 18 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 02b056ec63557..9d86911c8600d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -62,12 +62,6 @@ jobs: displayName: 'Linting' condition: true - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - ci/code_checks.sh code - displayName: 'Checks on imported code' - condition: true - - script: | export PATH=$HOME/miniconda3/bin:$PATH ci/code_checks.sh dependencies @@ -81,6 +75,12 @@ jobs: condition: true # Requires pandas + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/code_checks.sh code + displayName: 'Checks on imported code' + condition: true + - script: | export PATH=$HOME/miniconda3/bin:$PATH ci/code_checks.sh doctests diff --git a/ci/code_checks.sh b/ci/code_checks.sh index bb3500009f0c3..13eab6bd7c256 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -17,7 +17,7 @@ # $ ./ci/code_checks.sh docstrings # validate docstring errors # $ ./ci/code_checks.sh dependencies # check that dependencies are consistent -[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "dependecies" ]] || \ +[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "dependencies" ]] || \ { echo "Unknown command $1. Usage: $0 [lint|patterns|code|doctests|docstrings|dependencies]"; exit 9999; } source activate pandas diff --git a/ci/deps/travis-36.yaml b/ci/deps/travis-36.yaml index 1880fa2501581..599c7e3f30628 100644 --- a/ci/deps/travis-36.yaml +++ b/ci/deps/travis-36.yaml @@ -3,20 +3,11 @@ channels: - defaults - conda-forge dependencies: - - beautifulsoup4 - cython>=0.28.2 - dask - fastparquet - - flake8>=3.5 - - flake8-comprehensions - - flake8-rst=0.4.2 - gcsfs - geopandas - - html5lib - - ipython - - isort - - jinja2 - - lxml - matplotlib - nomkl - numexpr @@ -32,7 +23,6 @@ dependencies: - s3fs - scikit-learn - scipy - - seaborn - sqlalchemy - statsmodels - xarray @@ -48,6 +38,5 @@ dependencies: - pip: - brotlipy - coverage - - cpplint - pandas-datareader - python-dateutil diff --git a/environment.yml b/environment.yml index fc35f1290f1b1..918da32920396 100644 --- a/environment.yml +++ b/environment.yml @@ -52,3 +52,5 @@ dependencies: - xlrd - xlsxwriter - xlwt + - pip: + - cpplint From 7cf2d68ede0e7cc47f770bd2634c75578c945b45 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 20 Nov 2018 21:11:24 +0000 Subject: [PATCH 060/104] Addapting generate_pip_deps_from_conda.py to support pip dependencies in environment.yml --- requirements-dev.txt | 3 ++- scripts/generate_pip_deps_from_conda.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d01a21ac5fed5..ac48ef620e4a9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -40,4 +40,5 @@ statsmodels xarray xlrd xlsxwriter -xlwt \ No newline at end of file +xlwt +cpplint \ No newline at end of file diff --git a/scripts/generate_pip_deps_from_conda.py b/scripts/generate_pip_deps_from_conda.py index c11adc17bd33a..7b6eb1f9a32b5 100755 --- a/scripts/generate_pip_deps_from_conda.py +++ b/scripts/generate_pip_deps_from_conda.py @@ -75,7 +75,18 @@ def main(conda_fname, pip_fname, compare=False): with open(conda_fname) as conda_fd: deps = yaml.safe_load(conda_fd)['dependencies'] - pip_content = '\n'.join(filter(None, map(conda_package_to_pip, deps))) + pip_deps = [] + for dep in deps: + if isinstance(dep, str): + conda_dep = conda_package_to_pip(dep) + if conda_dep: + pip_deps.append(conda_dep) + elif isinstance(dep, dict) and len(dep) == 1 and 'pip' in dep: + pip_deps += dep['pip'] + else: + raise ValueError('Unexpected dependency {}'.format(dep)) + + pip_content = '\n'.join(pip_deps) if compare: with open(pip_fname) as pip_fd: From 2f8441eebfc5ddf5db293d9f841a7b64e9e9d382 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 20 Nov 2018 21:14:06 +0000 Subject: [PATCH 061/104] Setting DISPLAY env to see if that fixes the to_clipboard problem --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9d86911c8600d..c6f2a3271fa5f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -83,6 +83,7 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH + export DISPLAY=":99.0" # XXX just testing if this is why to_clipboard is failing ci/code_checks.sh doctests displayName: 'Running doctests' condition: true From 30ba23e67b376a617e5d4cd37bc9a206092b5173 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 20 Nov 2018 22:43:28 +0000 Subject: [PATCH 062/104] Consistent way to source the conda environment --- .travis.yml | 3 ++- azure-pipelines.yml | 17 ++++++++++++----- ci/code_checks.sh | 1 - 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0181dfe215bdc..95c3ca2e5cf30 100644 --- a/.travis.yml +++ b/.travis.yml @@ -100,7 +100,8 @@ install: before_script: - ci/install_db_travis.sh - - export DISPLAY=":99.0" + # XXX checking if this is still required now that doctests are in azure + # - export DISPLAY=":99.0" - ci/before_script_travis.sh script: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c6f2a3271fa5f..28faf6a55f552 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -41,9 +41,10 @@ jobs: echo '##vso[task.setvariable variable=AZURE]true' displayName: 'Setting environment variables' - # Does not require a conda environment + # Do not require a conda environment - script: | export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev ci/code_checks.sh patterns displayName: 'Looking for unwanted patterns' condition: true @@ -55,15 +56,17 @@ jobs: ci/incremental/setup_conda_environment.sh displayName: 'Set up environment' - # Does not require pandas + # Do not require pandas - script: | export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev ci/code_checks.sh lint displayName: 'Linting' condition: true - script: | export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev ci/code_checks.sh dependencies displayName: 'Dependencies consistency' condition: true @@ -74,29 +77,33 @@ jobs: displayName: 'Build' condition: true - # Requires pandas + # Require pandas - script: | export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev ci/code_checks.sh code displayName: 'Checks on imported code' condition: true - script: | export PATH=$HOME/miniconda3/bin:$PATH - export DISPLAY=":99.0" # XXX just testing if this is why to_clipboard is failing + source activate pandas-dev ci/code_checks.sh doctests displayName: 'Running doctests' condition: true + env: + DISPLAY: ":99.0" # required by to_clipboard doctests - script: | export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev ci/code_checks.sh docstrings displayName: 'Docstring validation' condition: true - script: | export PATH=$HOME/miniconda3/bin:$PATH - source activate pandas + source activate pandas-dev doc/make.py html displayName: 'Building docs' condition: true diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 13eab6bd7c256..304aafcb15adf 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -20,7 +20,6 @@ [[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "dependencies" ]] || \ { echo "Unknown command $1. Usage: $0 [lint|patterns|code|doctests|docstrings|dependencies]"; exit 9999; } -source activate pandas BASE_DIR="$(dirname $0)/.." RET=0 CHECK=$1 From 85172b3cf4d09e53e05ba413cfcabed15001ddc6 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 20 Nov 2018 22:56:33 +0000 Subject: [PATCH 063/104] Fixing typo in environment name, and removing pyqt debug info --- azure-pipelines.yml | 2 +- ci/incremental/setup_conda_environment.sh | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 28faf6a55f552..22aa0b0f988f9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -36,7 +36,7 @@ jobs: - script: | # XXX not sure why setting $PATH here does not work # echo '##vso[task.setvariable variable=PATH]$HOME/miniconda3/bin:$PATH' - echo '##vso[task.setvariable variable=CONDA_ENV]pandas' + echo '##vso[task.setvariable variable=CONDA_ENV]pandas-dev' echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' echo '##vso[task.setvariable variable=AZURE]true' displayName: 'Setting environment variables' diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index 4d9bca18f87e9..f3ac99d5e7c5a 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -16,15 +16,7 @@ conda remove --all -q -y -n $CONDA_ENV echo echo "[create env]" -echo $ENV_FILE -cat $ENV_FILE time conda env create -q -n "${CONDA_ENV}" --file="${ENV_FILE}" || exit 1 -conda list -e - -echo "Installing pyqt, as seems it's not got from the file (debugging why)" -conda install -yq pyqt -conda list -e - # Activate first set +v @@ -46,8 +38,6 @@ if [ -n "$LOCALE_OVERRIDE" ]; then sudo locale-gen "$LOCALE_OVERRIDE" fi -conda list -e - # # Install the compiler toolchain # if [[ $(uname) == Linux ]]; then # if [[ "$CONDA_SUBDIR" == "linux-32" || "$BITS32" == "yes" ]] ; then From 450f84a55851a8612ab56555d36dd78833c2f765 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 20 Nov 2018 23:24:25 +0000 Subject: [PATCH 064/104] Adding checkpoints to see where the docstests change the exit status --- ci/code_checks.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 304aafcb15adf..b3dc099290a6e 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -173,21 +173,33 @@ fi ### DOCTESTS ### if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then + echo "checkpoint 1" + echo $RET + MSG='Doctests frame.py' ; echo $MSG pytest -q --doctest-modules pandas/core/frame.py \ -k"-axes -combine -itertuples -join -nunique -pivot_table -quantile -query -reindex -reindex_axis -replace -round -set_index -stack -to_stata" RET=$(($RET + $?)) ; echo $MSG "DONE" + echo "checkpoint 2" + echo $RET + MSG='Doctests series.py' ; echo $MSG pytest -q --doctest-modules pandas/core/series.py \ -k"-nonzero -reindex -searchsorted -to_dict" RET=$(($RET + $?)) ; echo $MSG "DONE" + echo "checkpoint 3" + echo $RET + MSG='Doctests generic.py' ; echo $MSG pytest -q --doctest-modules pandas/core/generic.py \ -k"-_set_axis_name -_xs -describe -droplevel -groupby -interpolate -pct_change -pipe -reindex -reindex_axis -to_json -transpose -values -xs" RET=$(($RET + $?)) ; echo $MSG "DONE" + echo "checkpoint 4" + echo $RET + MSG='Doctests top-level reshaping functions' ; echo $MSG pytest -q --doctest-modules \ pandas/core/reshape/concat.py \ @@ -197,6 +209,9 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then -k"-crosstab -pivot_table -cut" RET=$(($RET + $?)) ; echo $MSG "DONE" + echo "checkpoint 5" + echo $RET + MSG='Doctests interval classes' ; echo $MSG pytest --doctest-modules -v \ pandas/core/indexes/interval.py \ @@ -204,6 +219,9 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then -k"-from_arrays -from_breaks -from_intervals -from_tuples -get_loc -set_closed -to_tuples -interval_range" RET=$(($RET + $?)) ; echo $MSG "DONE" + echo "checkpoint 6" + echo $RET + fi ### DOCSTRINGS ### From 7b4b1ea27e481d4809037374893c03925e8fd597 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 20 Nov 2018 23:26:09 +0000 Subject: [PATCH 065/104] Restoring required html5lib dependency --- ci/deps/travis-36.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/deps/travis-36.yaml b/ci/deps/travis-36.yaml index 599c7e3f30628..55f670193dacd 100644 --- a/ci/deps/travis-36.yaml +++ b/ci/deps/travis-36.yaml @@ -8,6 +8,7 @@ dependencies: - fastparquet - gcsfs - geopandas + - html5lib - matplotlib - nomkl - numexpr From 5d69e8b49c339ac9e0d3655f226698b459608b6b Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 20 Nov 2018 23:29:08 +0000 Subject: [PATCH 066/104] Removing travis doc job --- .travis.yml | 7 ------ ci/deps/travis-36-doc.yaml | 44 -------------------------------------- 2 files changed, 51 deletions(-) delete mode 100644 ci/deps/travis-36-doc.yaml diff --git a/.travis.yml b/.travis.yml index 95c3ca2e5cf30..c50271392e519 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,17 +66,10 @@ matrix: env: - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" SLOW=true - # In allow_failures - - dist: trusty - env: - - JOB="3.6, doc" ENV_FILE="ci/deps/travis-36-doc.yaml" DOC=true allow_failures: - dist: trusty env: - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" SLOW=true - - dist: trusty - env: - - JOB="3.6, doc" ENV_FILE="ci/deps/travis-36-doc.yaml" DOC=true before_install: - echo "before_install" diff --git a/ci/deps/travis-36-doc.yaml b/ci/deps/travis-36-doc.yaml deleted file mode 100644 index fb54c784d6fac..0000000000000 --- a/ci/deps/travis-36-doc.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: pandas -channels: - - defaults - - conda-forge -dependencies: - - beautifulsoup4 - - bottleneck - - cython>=0.28.2 - - fastparquet - - gitpython - - html5lib - - hypothesis>=3.58.0 - - ipykernel - - ipython - - ipywidgets - - lxml - - matplotlib - - nbconvert - - nbformat - - nbsphinx - - notebook - - numexpr - - numpy=1.13* - - openpyxl - - pandoc - - pyarrow - - pyqt - - pytables - - python-dateutil - - python-snappy - - python=3.6* - - pytz - - scipy - - seaborn - - sphinx - - sqlalchemy - - statsmodels - - xarray - - xlrd - - xlsxwriter - - xlwt - # universal - - pytest - - pytest-xdist From 3ce1aa0c23b74526ab591e960d0a46c1d4c5d6fe Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 20 Nov 2018 23:36:21 +0000 Subject: [PATCH 067/104] Removing linting comments, and token for the docs from travis --- .travis.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index c50271392e519..0140688b8e492 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,13 +14,6 @@ cache: - $HOME/.cache # cython cache - $HOME/.ccache # compiler cache -env: - global: - # create a github personal access token - # cd pandas-dev/pandas - # travis encrypt 'PANDAS_GH_TOKEN=personal_access_token' -r pandas-dev/pandas - - secure: "EkWLZhbrp/mXJOx38CHjs7BnjXafsqHtwxPQrqWy457VDFWhIY1DMnIR/lOWG+a20Qv52sCsFtiZEmMfUjf0pLGXOqurdxbYBGJ7/ikFLk9yV2rDwiArUlVM9bWFnFxHvdz9zewBH55WurrY4ShZWyV+x2dWjjceWG5VpWeI6sA=" - git: # for cloning depth: 1500 @@ -52,7 +45,7 @@ matrix: - python-gtk2 - dist: trusty env: - - JOB="3.6, lint, coverage" ENV_FILE="ci/deps/travis-36.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true LINT=true + - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true - dist: trusty env: - JOB="3.7, NumPy dev" ENV_FILE="ci/deps/travis-37-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network -W error" PANDAS_TESTING_MODE="deprecate" From ee01df4ce932eb2f85ed9b445ab288cfda5c918e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 20 Nov 2018 23:58:04 +0000 Subject: [PATCH 068/104] removing checkpoints and increasing verbosity of failing doctests --- ci/code_checks.sh | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index b3dc099290a6e..f1d259b4ac92c 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -173,33 +173,21 @@ fi ### DOCTESTS ### if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then - echo "checkpoint 1" - echo $RET - MSG='Doctests frame.py' ; echo $MSG pytest -q --doctest-modules pandas/core/frame.py \ -k"-axes -combine -itertuples -join -nunique -pivot_table -quantile -query -reindex -reindex_axis -replace -round -set_index -stack -to_stata" RET=$(($RET + $?)) ; echo $MSG "DONE" - echo "checkpoint 2" - echo $RET - MSG='Doctests series.py' ; echo $MSG pytest -q --doctest-modules pandas/core/series.py \ -k"-nonzero -reindex -searchsorted -to_dict" RET=$(($RET + $?)) ; echo $MSG "DONE" - echo "checkpoint 3" - echo $RET - MSG='Doctests generic.py' ; echo $MSG - pytest -q --doctest-modules pandas/core/generic.py \ + pytest -v --doctest-modules pandas/core/generic.py \ -k"-_set_axis_name -_xs -describe -droplevel -groupby -interpolate -pct_change -pipe -reindex -reindex_axis -to_json -transpose -values -xs" RET=$(($RET + $?)) ; echo $MSG "DONE" - echo "checkpoint 4" - echo $RET - MSG='Doctests top-level reshaping functions' ; echo $MSG pytest -q --doctest-modules \ pandas/core/reshape/concat.py \ @@ -209,9 +197,6 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then -k"-crosstab -pivot_table -cut" RET=$(($RET + $?)) ; echo $MSG "DONE" - echo "checkpoint 5" - echo $RET - MSG='Doctests interval classes' ; echo $MSG pytest --doctest-modules -v \ pandas/core/indexes/interval.py \ @@ -219,9 +204,6 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then -k"-from_arrays -from_breaks -from_intervals -from_tuples -get_loc -set_closed -to_tuples -interval_range" RET=$(($RET + $?)) ; echo $MSG "DONE" - echo "checkpoint 6" - echo $RET - fi ### DOCSTRINGS ### From e353e8ae1b08d82493e74daf9b612fa2d08c0306 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 21 Nov 2018 00:27:15 +0000 Subject: [PATCH 069/104] Restoring bf4 in travis deps, as it's required, and fixing the to_clipboard doctests problems by skipping it --- azure-pipelines.yml | 2 -- ci/code_checks.sh | 2 +- ci/deps/travis-36.yaml | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 22aa0b0f988f9..50241ed806df9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -91,8 +91,6 @@ jobs: ci/code_checks.sh doctests displayName: 'Running doctests' condition: true - env: - DISPLAY: ":99.0" # required by to_clipboard doctests - script: | export PATH=$HOME/miniconda3/bin:$PATH diff --git a/ci/code_checks.sh b/ci/code_checks.sh index f1d259b4ac92c..aace681d7528a 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -185,7 +185,7 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then MSG='Doctests generic.py' ; echo $MSG pytest -v --doctest-modules pandas/core/generic.py \ - -k"-_set_axis_name -_xs -describe -droplevel -groupby -interpolate -pct_change -pipe -reindex -reindex_axis -to_json -transpose -values -xs" + -k"-_set_axis_name -_xs -describe -droplevel -groupby -interpolate -pct_change -pipe -reindex -reindex_axis -to_json -transpose -values -xs -to_clipboard" RET=$(($RET + $?)) ; echo $MSG "DONE" MSG='Doctests top-level reshaping functions' ; echo $MSG diff --git a/ci/deps/travis-36.yaml b/ci/deps/travis-36.yaml index 55f670193dacd..f67fa1d5713d6 100644 --- a/ci/deps/travis-36.yaml +++ b/ci/deps/travis-36.yaml @@ -3,6 +3,7 @@ channels: - defaults - conda-forge dependencies: + - beautifulsoup4 - cython>=0.28.2 - dask - fastparquet From 92a3921fbec3d7adc08251818c806c4ef0310e08 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 21 Nov 2018 00:36:32 +0000 Subject: [PATCH 070/104] Moving scripts unit tests to azure --- azure-pipelines.yml | 5 +++++ ci/script_single.sh | 2 -- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 50241ed806df9..e32ffeec08f58 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -99,6 +99,11 @@ jobs: displayName: 'Docstring validation' condition: true + - script: | + pytest --capture=no --strict scripts + displayName: 'Testing docstring validaton script' + condition: true + - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev diff --git a/ci/script_single.sh b/ci/script_single.sh index bc34a62a7d6b4..fff019143bd73 100755 --- a/ci/script_single.sh +++ b/ci/script_single.sh @@ -25,8 +25,6 @@ fi if [ "$COVERAGE" ]; then echo pytest -s -m "single" --durations=10 --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=test-data-single.xml $TEST_ARGS pandas pytest -s -m "single" --durations=10 --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=test-data-single.xml $TEST_ARGS pandas - echo pytest -s --strict scripts - pytest -s --strict scripts else echo pytest -m "single" --durations=10 --junitxml=test-data-single.xml --strict $TEST_ARGS pandas pytest -m "single" --durations=10 --junitxml=test-data-single.xml --strict $TEST_ARGS pandas From 9e34037fe0bc749fefa401a7f9cfd89f444e29c2 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 21 Nov 2018 01:10:04 +0000 Subject: [PATCH 071/104] Adding conda environment to run scripts tests, and changing verbosity of doctests --- azure-pipelines.yml | 2 ++ ci/code_checks.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e32ffeec08f58..eef05592fef14 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -100,6 +100,8 @@ jobs: condition: true - script: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev pytest --capture=no --strict scripts displayName: 'Testing docstring validaton script' condition: true diff --git a/ci/code_checks.sh b/ci/code_checks.sh index aace681d7528a..ef5095417d6f8 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -184,7 +184,7 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then RET=$(($RET + $?)) ; echo $MSG "DONE" MSG='Doctests generic.py' ; echo $MSG - pytest -v --doctest-modules pandas/core/generic.py \ + pytest -q --doctest-modules pandas/core/generic.py \ -k"-_set_axis_name -_xs -describe -droplevel -groupby -interpolate -pct_change -pipe -reindex -reindex_axis -to_json -transpose -values -xs -to_clipboard" RET=$(($RET + $?)) ; echo $MSG "DONE" From 5a82f5916c29168d93a535ba82b49698e3a29a1b Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 21 Nov 2018 14:37:17 +0000 Subject: [PATCH 072/104] Using connection string secret variable instead of composing it in the config --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index eef05592fef14..c9dbb700b6e89 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -120,7 +120,7 @@ jobs: condition: true env: SOURCE_PATH: $(Build.SourcesDirectory)/doc/build/html/ - AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=https;AccountName=$(AZURE_STORAGE_ACCOUNT);AccountKey=$(AZURE_STORAGE_ACCESS_KEY)" + AZURE_STORAGE_CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING) - script: | if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then From 6cbb31b3a9731e6822665093b4c53ed6c5cd4a79 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 22 Nov 2018 01:35:57 +0000 Subject: [PATCH 073/104] Adding benchmark run to the CI --- asv_bench/benchmarks/algorithms.py | 3 ++- azure-pipelines.yml | 19 +++++++++++++++++++ environment.yml | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/asv_bench/benchmarks/algorithms.py b/asv_bench/benchmarks/algorithms.py index 1ab88dc9f9e6d..7dbf9dea4014f 100644 --- a/asv_bench/benchmarks/algorithms.py +++ b/asv_bench/benchmarks/algorithms.py @@ -1,7 +1,8 @@ import warnings from importlib import import_module -import numpy as np +# making the benchmarks fail, to see if this is detected by the CI: +# import numpy as np import pandas as pd from pandas.util import testing as tm diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c9dbb700b6e89..beb2fabc92540 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -144,3 +144,22 @@ jobs: condition: true env: REPO_DIR: $(Build.ArtifactStagingDirectory)/pandas-docs-travis + + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev + if git diff upstream/master --name-only | grep -q "^asv_bench/"; then + cd asv_bench + ASV_ERRORS=$(asv dev > /dev/null) + if [[ $ASV_ERRORS ]]; then + echo "Benchmarks reported errors:" + echo $ASV_ERRORS + exit 1 + else + echo "Benchmark run without errors" + fi + else + echo "Benchmark not running, no changes detected" + fi + displayName: 'Running benchmarks' + condition: true diff --git a/environment.yml b/environment.yml index 918da32920396..99db4f561c0f3 100644 --- a/environment.yml +++ b/environment.yml @@ -10,6 +10,7 @@ dependencies: - pytz # development + - asv - Cython>=0.28.2 - flake8 - flake8-comprehensions From 75ad473946a89cc4d509e81208c5c4fc19db5cf8 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 22 Nov 2018 09:58:26 +0000 Subject: [PATCH 074/104] Adding git remote for the benchmarks, and removing connection string reassignment --- azure-pipelines.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index beb2fabc92540..3fe4201024935 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -120,7 +120,9 @@ jobs: condition: true env: SOURCE_PATH: $(Build.SourcesDirectory)/doc/build/html/ - AZURE_STORAGE_CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING) + # removing, to see if the job picks it directly, and the problem is + # in assigning a variable to itself + # AZURE_STORAGE_CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING) - script: | if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then @@ -148,6 +150,7 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev + git remote add upstream https://github.com/pandas-dev/pandas.git if git diff upstream/master --name-only | grep -q "^asv_bench/"; then cd asv_bench ASV_ERRORS=$(asv dev > /dev/null) From 7af04e143fc8b132f4e96f2d9f376ebe10f72987 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 22 Nov 2018 10:59:25 +0000 Subject: [PATCH 075/104] Providing azure storage info as arguments instead of env, and fetching from upstream before the benchmarks, and printing debug information --- azure-pipelines.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3fe4201024935..f331c48cf24e9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -115,14 +115,15 @@ jobs: - script: | az extension add --name storage-preview - az storage blob upload-batch -s $SOURCE_PATH -d '$web' + az storage blob upload-batch --account-name=$(AZURE_STORAGE_ACCOUNT) --account-key=$(AZURE_STORAGE_ACCESS_KEY) -s $SOURCE_PATH -d '$web' displayName: 'Publishing docs (Azure storage)' condition: true env: SOURCE_PATH: $(Build.SourcesDirectory)/doc/build/html/ - # removing, to see if the job picks it directly, and the problem is - # in assigning a variable to itself + # Moving connection data to arguments # AZURE_STORAGE_CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING) + # $(AZURE_STORAGE_ACCOUNT) + # $(AZURE_STORAGE_ACCESS_KEY) - script: | if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then @@ -150,7 +151,14 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev + git remote -v + git branch + git branch -a git remote add upstream https://github.com/pandas-dev/pandas.git + git fetch upstream + git remote -v + git branch + git branch -a if git diff upstream/master --name-only | grep -q "^asv_bench/"; then cd asv_bench ASV_ERRORS=$(asv dev > /dev/null) From cc4331c46ca7eb412232de0531872f80d5971809 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 22 Nov 2018 11:54:56 +0000 Subject: [PATCH 076/104] Trying to fix benchmarks and azure storage issues --- azure-pipelines.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f331c48cf24e9..987d6485fac30 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -115,15 +115,15 @@ jobs: - script: | az extension add --name storage-preview - az storage blob upload-batch --account-name=$(AZURE_STORAGE_ACCOUNT) --account-key=$(AZURE_STORAGE_ACCESS_KEY) -s $SOURCE_PATH -d '$web' + az storage blob upload-batch --account-name=$ACCOUNT_NAME --account-key=$ACCOUNT_KEY -s $SOURCE_PATH -d '$web' displayName: 'Publishing docs (Azure storage)' condition: true env: SOURCE_PATH: $(Build.SourcesDirectory)/doc/build/html/ # Moving connection data to arguments # AZURE_STORAGE_CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING) - # $(AZURE_STORAGE_ACCOUNT) - # $(AZURE_STORAGE_ACCESS_KEY) + ACCOUNT_NAME: $(AZURE_STORAGE_ACCOUNT) + ACCOUNT_KEY: $(AZURE_STORAGE_ACCESS_KEY) - script: | if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then @@ -151,19 +151,16 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev - git remote -v - git branch - git branch -a git remote add upstream https://github.com/pandas-dev/pandas.git git fetch upstream - git remote -v - git branch - git branch -a if git diff upstream/master --name-only | grep -q "^asv_bench/"; then cd asv_bench + asv dev ASV_ERRORS=$(asv dev > /dev/null) + echo "Captured ASV_ERRORS:" + echo $ASV_ERRORS if [[ $ASV_ERRORS ]]; then - echo "Benchmarks reported errors:" + echo "##vso[task.logissue type=error]Benchmarks failed to run" echo $ASV_ERRORS exit 1 else From c5df4011b7a1545693bbd01b3d87cd7ed4a37beb Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 22 Nov 2018 14:30:20 +0000 Subject: [PATCH 077/104] Testing if azure storage key has a missing =, and setting up machine for asv --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 987d6485fac30..b8f4f29434bf1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -123,7 +123,7 @@ jobs: # Moving connection data to arguments # AZURE_STORAGE_CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING) ACCOUNT_NAME: $(AZURE_STORAGE_ACCOUNT) - ACCOUNT_KEY: $(AZURE_STORAGE_ACCESS_KEY) + ACCOUNT_KEY: "$(AZURE_STORAGE_ACCESS_KEY)=" - script: | if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then @@ -155,6 +155,7 @@ jobs: git fetch upstream if git diff upstream/master --name-only | grep -q "^asv_bench/"; then cd asv_bench + asv machine --yes asv dev ASV_ERRORS=$(asv dev > /dev/null) echo "Captured ASV_ERRORS:" From 4eac8bb59329deff06d6bc26d8c5e00897c22ff6 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 22 Nov 2018 17:39:06 +0000 Subject: [PATCH 078/104] More changes to the doc upload and the benchmarks, to try to make them work --- azure-pipelines.yml | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b8f4f29434bf1..ff637e1065bbb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -27,12 +27,8 @@ jobs: - job: 'Checks_and_doc' pool: vmImage: ubuntu-16.04 + timeoutInMinutes: 90 steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - architecture: 'x64' - - script: | # XXX not sure why setting $PATH here does not work # echo '##vso[task.setvariable variable=PATH]$HOME/miniconda3/bin:$PATH' @@ -44,7 +40,6 @@ jobs: # Do not require a conda environment - script: | export PATH=$HOME/miniconda3/bin:$PATH - source activate pandas-dev ci/code_checks.sh patterns displayName: 'Looking for unwanted patterns' condition: true @@ -114,16 +109,23 @@ jobs: condition: true - script: | + if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then + export NAME="master" + else + export NAME=$(System.PullRequest.PullRequestId) + fi + echo $CONNECTION_STRING | wc -c az extension add --name storage-preview - az storage blob upload-batch --account-name=$ACCOUNT_NAME --account-key=$ACCOUNT_KEY -s $SOURCE_PATH -d '$web' + az storage blob upload-batch --connection-string $CONNECTION_STRING \ + --source $SOURCE \ + --destination $DESTINATION \ + --destination-path $NAME displayName: 'Publishing docs (Azure storage)' condition: true env: - SOURCE_PATH: $(Build.SourcesDirectory)/doc/build/html/ - # Moving connection data to arguments - # AZURE_STORAGE_CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING) - ACCOUNT_NAME: $(AZURE_STORAGE_ACCOUNT) - ACCOUNT_KEY: "$(AZURE_STORAGE_ACCESS_KEY)=" + CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING) + SOURCE: $(Build.SourcesDirectory)/doc/build/html/ + DESTINATION: 'docs' - script: | if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then @@ -156,19 +158,16 @@ jobs: if git diff upstream/master --name-only | grep -q "^asv_bench/"; then cd asv_bench asv machine --yes - asv dev - ASV_ERRORS=$(asv dev > /dev/null) - echo "Captured ASV_ERRORS:" - echo $ASV_ERRORS - if [[ $ASV_ERRORS ]]; then - echo "##vso[task.logissue type=error]Benchmarks failed to run" + ASV_OUTPUT=$(asv dev) + if [[ $(echo $ASV_OUTPUT | grep "failure") ]]; then + echo "##vso[task.logissue type=error]Benchmarks run with errors" echo $ASV_ERRORS exit 1 else - echo "Benchmark run without errors" + echo "Benchmarks run without errors" fi else - echo "Benchmark not running, no changes detected" + echo "Benchmarks did not run, no changes detected" fi displayName: 'Running benchmarks' condition: true From 00032d12ac4bfb23cd51977a649f15ef846fc9bf Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 22 Nov 2018 19:22:49 +0000 Subject: [PATCH 079/104] More fixes to doc upload and benchmarks --- azure-pipelines.yml | 7 ++++--- requirements-dev.txt | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ff637e1065bbb..ecb6e3b344c60 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -114,12 +114,12 @@ jobs: else export NAME=$(System.PullRequest.PullRequestId) fi - echo $CONNECTION_STRING | wc -c az extension add --name storage-preview az storage blob upload-batch --connection-string $CONNECTION_STRING \ --source $SOURCE \ --destination $DESTINATION \ --destination-path $NAME + echo "Documentation uploaded to https://pandas.blob.core.windows.net/docs/$NAME" displayName: 'Publishing docs (Azure storage)' condition: true env: @@ -159,9 +159,10 @@ jobs: cd asv_bench asv machine --yes ASV_OUTPUT=$(asv dev) - if [[ $(echo $ASV_OUTPUT | grep "failure") ]]; then + echo "$ASV_OUTPUT" | grep "failure" # TODO remove this when it works + if [[ $(echo "$ASV_OUTPUT" | grep "failure") ]]; then echo "##vso[task.logissue type=error]Benchmarks run with errors" - echo $ASV_ERRORS + echo $ASV_OUTPUT exit 1 else echo "Benchmarks run without errors" diff --git a/requirements-dev.txt b/requirements-dev.txt index ac48ef620e4a9..7d0464b347c8e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,7 @@ NumPy python-dateutil>=2.5.0 pytz +asv Cython>=0.28.2 flake8 flake8-comprehensions From e9ab7541fe8e8ab26254eca47dd0edefa6cb991a Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 22 Nov 2018 20:16:23 +0000 Subject: [PATCH 080/104] Adding debug info for the docs upload and the benchmarks --- azure-pipelines.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ecb6e3b344c60..65cabe139ddbd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -109,6 +109,12 @@ jobs: condition: true - script: | + echo "checking different ways to access variables" + echo $(TEST_VARIABLE) + echo $(pandas.TEST_VARIABLE) + echo $TEST_VAR1 + echo $TEST_VAR2 + echo "DONE" if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then export NAME="master" else @@ -124,6 +130,8 @@ jobs: condition: true env: CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING) + TEST_VAR1: $(TEST_VARIABLE) + TEST_VAR2: $(pandas.TEST_VARIABLE) SOURCE: $(Build.SourcesDirectory)/doc/build/html/ DESTINATION: 'docs' @@ -158,8 +166,12 @@ jobs: if git diff upstream/master --name-only | grep -q "^asv_bench/"; then cd asv_bench asv machine --yes - ASV_OUTPUT=$(asv dev) + ASV_OUTPUT="$(asv dev)" + echo "all output" + echo "$ASV_OUTPUT" + echo "failure lines" echo "$ASV_OUTPUT" | grep "failure" # TODO remove this when it works + echo "end of debug info" if [[ $(echo "$ASV_OUTPUT" | grep "failure") ]]; then echo "##vso[task.logissue type=error]Benchmarks run with errors" echo $ASV_OUTPUT From e47b4e119ac4ecf5bd34beeb6f1b93eaa92defa5 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 22 Nov 2018 21:43:43 +0000 Subject: [PATCH 081/104] Fixing bug in benchmarks set up, and fixing docstring errors, so validation passes --- azure-pipelines.yml | 7 +------ pandas/core/frame.py | 4 ---- pandas/core/panel.py | 4 ++-- pandas/core/tools/timedeltas.py | 2 +- pandas/io/gbq.py | 12 ++++-------- pandas/io/json/json.py | 8 ++++---- 6 files changed, 12 insertions(+), 25 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 65cabe139ddbd..8bbcfd3759075 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -167,12 +167,7 @@ jobs: cd asv_bench asv machine --yes ASV_OUTPUT="$(asv dev)" - echo "all output" - echo "$ASV_OUTPUT" - echo "failure lines" - echo "$ASV_OUTPUT" | grep "failure" # TODO remove this when it works - echo "end of debug info" - if [[ $(echo "$ASV_OUTPUT" | grep "failure") ]]; then + if [[ $(echo "$ASV_OUTPUT" | grep "failed") ]]; then echo "##vso[task.logissue type=error]Benchmarks run with errors" echo $ASV_OUTPUT exit 1 diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 572bb3668caf8..5e1ef629013a6 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -1276,10 +1276,6 @@ def to_gbq(self, destination_table, project_id=None, chunksize=None, If table exists, drop it, recreate it, and insert data. ``'append'`` If table exists, insert data. Create if does not exist. - private_key : str, optional - Service account private key in JSON format. Can be file path - or string contents. This is useful for remote server - authentication (eg. Jupyter/IPython notebook on remote host). auth_local_webserver : bool, default False Use the `local webserver flow`_ instead of the `console flow`_ when getting user credentials. diff --git a/pandas/core/panel.py b/pandas/core/panel.py index 5ae7848b5adc6..5940e5b2b106a 100644 --- a/pandas/core/panel.py +++ b/pandas/core/panel.py @@ -124,10 +124,10 @@ class Panel(NDFrame): axis=1 minor_axis : Index or array-like axis=2 - dtype : dtype, default None - Data type to force, otherwise infer copy : boolean, default False Copy data from inputs. Only affects DataFrame / 2d ndarray input + dtype : dtype, default None + Data type to force, otherwise infer """ @property diff --git a/pandas/core/tools/timedeltas.py b/pandas/core/tools/timedeltas.py index 7b0a3da738436..6bcf56c306e6a 100644 --- a/pandas/core/tools/timedeltas.py +++ b/pandas/core/tools/timedeltas.py @@ -50,7 +50,7 @@ def to_timedelta(arg, unit='ns', box=True, errors='raise'): timedelta64 or numpy.array of timedelta64 Output type returned if parsing succeeded. - See also + See Also -------- DataFrame.astype : Cast argument to a specified dtype. to_datetime : Convert argument to datetime. diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index 4d5b2fda7cd10..639b68d433ac6 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -52,10 +52,6 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, reauth : boolean, default False Force Google BigQuery to re-authenticate the user. This is useful if multiple accounts are used. - private_key : str, optional - Service account private key in JSON format. Can be file path - or string contents. This is useful for remote server - authentication (eg. Jupyter/IPython notebook on remote host). auth_local_webserver : boolean, default False Use the `local webserver flow`_ instead of the `console flow`_ when getting user credentials. @@ -107,10 +103,6 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, *New in version 0.8.0 of pandas-gbq*. .. versionadded:: 0.24.0 - verbose : None, deprecated - Deprecated in pandas-gbq version 0.4.0. Use the `logging module to - adjust verbosity instead - `__. private_key : str, deprecated Deprecated in pandas-gbq version 0.8.0. Use the ``credentials`` parameter and @@ -122,6 +114,10 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, Service account private key in JSON format. Can be file path or string contents. This is useful for remote server authentication (eg. Jupyter/IPython notebook on remote host). + verbose : None, deprecated + Deprecated in pandas-gbq version 0.4.0. Use the `logging module to + adjust verbosity instead + `__. Returns ------- diff --git a/pandas/io/json/json.py b/pandas/io/json/json.py index 38f8cd5412015..03568a7284411 100644 --- a/pandas/io/json/json.py +++ b/pandas/io/json/json.py @@ -310,13 +310,13 @@ def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True, is to try and detect the correct precision, but if this is not desired then pass one of 's', 'ms', 'us' or 'ns' to force parsing only seconds, milliseconds, microseconds or nanoseconds respectively. - lines : boolean, default False - Read the file as a json object per line. + encoding : str, default is 'utf-8' + The encoding to use to decode py3 bytes. .. versionadded:: 0.19.0 - encoding : str, default is 'utf-8' - The encoding to use to decode py3 bytes. + lines : boolean, default False + Read the file as a json object per line. .. versionadded:: 0.19.0 From a225d44fd5a08bcba2aa3d14fccd0bd6957c4a6f Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 23 Nov 2018 01:07:50 +0000 Subject: [PATCH 082/104] Clean up of debug info --- .travis.yml | 3 +-- asv_bench/benchmarks/algorithms.py | 3 +-- azure-pipelines.yml | 8 -------- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0140688b8e492..8d794442bd271 100644 --- a/.travis.yml +++ b/.travis.yml @@ -86,8 +86,7 @@ install: before_script: - ci/install_db_travis.sh - # XXX checking if this is still required now that doctests are in azure - # - export DISPLAY=":99.0" + - export DISPLAY=":99.0" - ci/before_script_travis.sh script: diff --git a/asv_bench/benchmarks/algorithms.py b/asv_bench/benchmarks/algorithms.py index 7dbf9dea4014f..1ab88dc9f9e6d 100644 --- a/asv_bench/benchmarks/algorithms.py +++ b/asv_bench/benchmarks/algorithms.py @@ -1,8 +1,7 @@ import warnings from importlib import import_module -# making the benchmarks fail, to see if this is detected by the CI: -# import numpy as np +import numpy as np import pandas as pd from pandas.util import testing as tm diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8bbcfd3759075..4bf12175c9108 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -109,12 +109,6 @@ jobs: condition: true - script: | - echo "checking different ways to access variables" - echo $(TEST_VARIABLE) - echo $(pandas.TEST_VARIABLE) - echo $TEST_VAR1 - echo $TEST_VAR2 - echo "DONE" if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then export NAME="master" else @@ -130,8 +124,6 @@ jobs: condition: true env: CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING) - TEST_VAR1: $(TEST_VARIABLE) - TEST_VAR2: $(pandas.TEST_VARIABLE) SOURCE: $(Build.SourcesDirectory)/doc/build/html/ DESTINATION: 'docs' From d6d5c6631acca69c6ea1fe99347d0b2080e17dc2 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 23 Nov 2018 01:15:03 +0000 Subject: [PATCH 083/104] Uploading docs to GitHub pages in the same way as travis --- azure-pipelines.yml | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4bf12175c9108..8737a6bb0276c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -129,26 +129,32 @@ jobs: - script: | if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then - export NAME="master" - else - export NAME=$(System.PullRequest.PullRequestId) + mkdir -p $REPO_DIR + rm -rf $REPO_DIR/* + cp -r doc/build/html/* $REPO_DIR/ + cd $REPO_DIR + git config --global user.email "pandas-docs-bot@localhost.foo" + git config --global user.name "pandas-docs-bot" + git init + touch README + git add README + git commit -m "Initial commit" --allow-empty + git branch gh-pages + git checkout gh-pages + touch .nojekyll + git add --all . + git commit -m "Version" --allow-empty + git remote remove origin + git remote add origin "https://${TOKEN}@github.com/pandas-dev/pandas-docs-travis.git" + git fetch origin + git remote -v + git push origin gh-pages -f fi - export PR_DIR=$REPO_DIR/pr/$NAME - git clone https://$(GITHUB_DOC_TOKEN)@github.com/pandas-dev/pandas-docs-travis.git $REPO_DIR - mkdir -p $PR_DIR - rm -rf $PR_DIR/* - cp -r doc/build/html/* $PR_DIR/ - cp LICENSE $PR_DIR/ - cd $REPO_DIR - git add --all . - git config user.email "pandas-docs-bot@localhost" - git config user.name "pandas-docs-bot" - git commit -m "Added pandas documentation for $NAME" - git push displayName: 'Publishing docs (GitHub pages)' condition: true env: REPO_DIR: $(Build.ArtifactStagingDirectory)/pandas-docs-travis + TOKEN: $(GITHUB_DOCS_TOKEN) - script: | export PATH=$HOME/miniconda3/bin:$PATH From f7a6048d51f4c002bb39c6fb8845f00fe8769833 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 23 Nov 2018 01:52:04 +0000 Subject: [PATCH 084/104] Adding variable group to the job config, to see if that makes them accessible --- azure-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8737a6bb0276c..3c5ee085d0d85 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -28,6 +28,8 @@ jobs: pool: vmImage: ubuntu-16.04 timeoutInMinutes: 90 + variables: + - group: pandas steps: - script: | # XXX not sure why setting $PATH here does not work From 9aa18d0c42d5ad05991ac1e2cd0532babd88a61a Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 23 Nov 2018 09:09:43 +0000 Subject: [PATCH 085/104] WIP: Restoring travis documentation build --- .travis.yml | 8 +++++++ ci/build_docs.sh | 52 ++++++++++++++++++++++++++++++++++++++++++++ ci/run_build_docs.sh | 10 +++++++++ 3 files changed, 70 insertions(+) create mode 100755 ci/build_docs.sh create mode 100755 ci/run_build_docs.sh diff --git a/.travis.yml b/.travis.yml index 8d794442bd271..eb28b821fff7b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,10 +59,17 @@ matrix: env: - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" SLOW=true + # In allow_failures + - dist: trusty + env: + - JOB="3.6, doc" ENV_FILE="ci/deps/travis-36-doc.yaml" DOC=true allow_failures: - dist: trusty env: - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" SLOW=true + - dist: trusty + env: + - JOB="3.6, doc" ENV_FILE="ci/deps/travis-36-doc.yaml" DOC=true before_install: - echo "before_install" @@ -91,6 +98,7 @@ before_script: script: - echo "script start" + - ci/run_build_docs.sh - ci/script_single.sh - ci/script_multi.sh diff --git a/ci/build_docs.sh b/ci/build_docs.sh new file mode 100755 index 0000000000000..33340a1c038dc --- /dev/null +++ b/ci/build_docs.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +if [ "${TRAVIS_OS_NAME}" != "linux" ]; then + echo "not doing build_docs on non-linux" + exit 0 +fi + +cd "$TRAVIS_BUILD_DIR"/doc +echo "inside $0" + +if [ "$DOC" ]; then + + echo "Will build docs" + + source activate pandas + + echo ############################### + echo # Log file for the doc build # + echo ############################### + + echo ./make.py + ./make.py + + echo ######################## + echo # Create and send docs # + echo ######################## + + cd build/html + git config --global user.email "pandas-docs-bot@localhost.foo" + git config --global user.name "pandas-docs-bot" + + # create the repo + git init + + touch README + git add README + git commit -m "Initial commit" --allow-empty + git branch gh-pages + git checkout gh-pages + touch .nojekyll + git add --all . + git commit -m "Version" --allow-empty + + git remote remove origin + git remote add origin "https://${PANDAS_GH_TOKEN}@github.com/pandas-dev/pandas-docs-travis.git" + git fetch origin + git remote -v + + git push origin gh-pages -f +fi + +exit 0 diff --git a/ci/run_build_docs.sh b/ci/run_build_docs.sh new file mode 100755 index 0000000000000..2909b9619552e --- /dev/null +++ b/ci/run_build_docs.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +echo "inside $0" + +"$TRAVIS_BUILD_DIR"/ci/build_docs.sh 2>&1 + +# wait until subprocesses finish (build_docs.sh) +wait + +exit 0 From 283233dbb85c7391fec0de0e359955018f23e559 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 23 Nov 2018 09:16:07 +0000 Subject: [PATCH 086/104] Restoing documentation build in travis --- .travis.yml | 7 ++++++ ci/deps/travis-36-doc.yaml | 44 ++++++++++++++++++++++++++++++++++++++ ci/script_multi.sh | 5 ++++- ci/script_single.sh | 5 ++++- 4 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 ci/deps/travis-36-doc.yaml diff --git a/.travis.yml b/.travis.yml index eb28b821fff7b..1dfa8465d3040 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,13 @@ cache: - $HOME/.cache # cython cache - $HOME/.ccache # compiler cache +env: + global: + # create a github personal access token + # cd pandas-dev/pandas + # travis encrypt 'PANDAS_GH_TOKEN=personal_access_token' -r pandas-dev/pandas + - secure: "EkWLZhbrp/mXJOx38CHjs7BnjXafsqHtwxPQrqWy457VDFWhIY1DMnIR/lOWG+a20Qv52sCsFtiZEmMfUjf0pLGXOqurdxbYBGJ7/ikFLk9yV2rDwiArUlVM9bWFnFxHvdz9zewBH55WurrY4ShZWyV+x2dWjjceWG5VpWeI6sA=" + git: # for cloning depth: 1500 diff --git a/ci/deps/travis-36-doc.yaml b/ci/deps/travis-36-doc.yaml new file mode 100644 index 0000000000000..fb54c784d6fac --- /dev/null +++ b/ci/deps/travis-36-doc.yaml @@ -0,0 +1,44 @@ +name: pandas +channels: + - defaults + - conda-forge +dependencies: + - beautifulsoup4 + - bottleneck + - cython>=0.28.2 + - fastparquet + - gitpython + - html5lib + - hypothesis>=3.58.0 + - ipykernel + - ipython + - ipywidgets + - lxml + - matplotlib + - nbconvert + - nbformat + - nbsphinx + - notebook + - numexpr + - numpy=1.13* + - openpyxl + - pandoc + - pyarrow + - pyqt + - pytables + - python-dateutil + - python-snappy + - python=3.6* + - pytz + - scipy + - seaborn + - sphinx + - sqlalchemy + - statsmodels + - xarray + - xlrd + - xlsxwriter + - xlwt + # universal + - pytest + - pytest-xdist diff --git a/ci/script_multi.sh b/ci/script_multi.sh index 2fbda3cdd2792..e56d5da7232b2 100755 --- a/ci/script_multi.sh +++ b/ci/script_multi.sh @@ -23,7 +23,10 @@ fi export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))') echo PYTHONHASHSEED=$PYTHONHASHSEED -if [ "$COVERAGE" ]; then +if [ "$DOC" ]; then + echo "We are not running pytest as this is a doc-build" + +elif [ "$COVERAGE" ]; then echo pytest -s -n 2 -m "not single" --durations=10 --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas pytest -s -n 2 -m "not single" --durations=10 --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas diff --git a/ci/script_single.sh b/ci/script_single.sh index fff019143bd73..f5a2d335a8f14 100755 --- a/ci/script_single.sh +++ b/ci/script_single.sh @@ -22,7 +22,10 @@ if echo "$TEST_ARGS" | grep -e --skip-network -q; then export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4; fi -if [ "$COVERAGE" ]; then +if [ "$DOC" ]; then + echo "We are not running pytest as this is a doc-build" + +elif [ "$COVERAGE" ]; then echo pytest -s -m "single" --durations=10 --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=test-data-single.xml $TEST_ARGS pandas pytest -s -m "single" --durations=10 --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=test-data-single.xml $TEST_ARGS pandas else From 991304d695bf77f6fa8a01f0cff8229d23c540c1 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 27 Nov 2018 21:51:15 +0000 Subject: [PATCH 087/104] Removing variable group, as variables are not in a group anymore, and not running publishing docs to github pages --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3c5ee085d0d85..4c7268e11f359 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -28,8 +28,6 @@ jobs: pool: vmImage: ubuntu-16.04 timeoutInMinutes: 90 - variables: - - group: pandas steps: - script: | # XXX not sure why setting $PATH here does not work @@ -130,6 +128,8 @@ jobs: DESTINATION: 'docs' - script: | + echo "Publishing documentation to GitHub pages is still done in Travis" + exit 0 if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then mkdir -p $REPO_DIR rm -rf $REPO_DIR/* From 19c396f576cf64d4e61b074c4973f0217d62cde0 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 27 Nov 2018 22:42:41 +0000 Subject: [PATCH 088/104] Adding missing conda environment activation --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4c7268e11f359..c99a08ff2f745 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -68,6 +68,7 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev ci/incremental/build.sh displayName: 'Build' condition: true From a69e667e9de3a57f51e5263c9f1ea5981926b526 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 27 Nov 2018 23:28:26 +0000 Subject: [PATCH 089/104] Fixing docstring errors --- pandas/core/window.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pandas/core/window.py b/pandas/core/window.py index a079cea0fabd1..e1350c1fff34a 100644 --- a/pandas/core/window.py +++ b/pandas/core/window.py @@ -31,14 +31,14 @@ _shared_docs = dict(**_shared_docs) _doc_template = """ -Returns -------- -same type as input - -See Also --------- -pandas.Series.%(name)s -pandas.DataFrame.%(name)s + Returns + ------- + same type as input + + See Also + -------- + Series.%(name)s + DataFrame.%(name)s """ From 59d55d87b5564c606e05a9d646a1d15d7f22052c Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 28 Nov 2018 16:48:27 +0000 Subject: [PATCH 090/104] Removing documentation build from azure --- azure-pipelines.yml | 69 +-------------------------------------------- 1 file changed, 1 insertion(+), 68 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c99a08ff2f745..c29b0eba6da80 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -30,8 +30,7 @@ jobs: timeoutInMinutes: 90 steps: - script: | - # XXX not sure why setting $PATH here does not work - # echo '##vso[task.setvariable variable=PATH]$HOME/miniconda3/bin:$PATH' + echo '##vso[task.prependpath]$HOME/miniconda3/bin' echo '##vso[task.setvariable variable=CONDA_ENV]pandas-dev' echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' echo '##vso[task.setvariable variable=AZURE]true' @@ -39,13 +38,11 @@ jobs: # Do not require a conda environment - script: | - export PATH=$HOME/miniconda3/bin:$PATH ci/code_checks.sh patterns displayName: 'Looking for unwanted patterns' condition: true - script: | - export PATH=$HOME/miniconda3/bin:$PATH sudo apt-get install -y libc6-dev-i386 ci/incremental/install_miniconda.sh ci/incremental/setup_conda_environment.sh @@ -53,21 +50,18 @@ jobs: # Do not require pandas - script: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh lint displayName: 'Linting' condition: true - script: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh dependencies displayName: 'Dependencies consistency' condition: true - script: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/incremental/build.sh displayName: 'Build' @@ -75,90 +69,29 @@ jobs: # Require pandas - script: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh code displayName: 'Checks on imported code' condition: true - script: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh doctests displayName: 'Running doctests' condition: true - script: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh docstrings displayName: 'Docstring validation' condition: true - script: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev pytest --capture=no --strict scripts displayName: 'Testing docstring validaton script' condition: true - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - source activate pandas-dev - doc/make.py html - displayName: 'Building docs' - condition: true - - - script: | - if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then - export NAME="master" - else - export NAME=$(System.PullRequest.PullRequestId) - fi - az extension add --name storage-preview - az storage blob upload-batch --connection-string $CONNECTION_STRING \ - --source $SOURCE \ - --destination $DESTINATION \ - --destination-path $NAME - echo "Documentation uploaded to https://pandas.blob.core.windows.net/docs/$NAME" - displayName: 'Publishing docs (Azure storage)' - condition: true - env: - CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING) - SOURCE: $(Build.SourcesDirectory)/doc/build/html/ - DESTINATION: 'docs' - - - script: | - echo "Publishing documentation to GitHub pages is still done in Travis" - exit 0 - if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then - mkdir -p $REPO_DIR - rm -rf $REPO_DIR/* - cp -r doc/build/html/* $REPO_DIR/ - cd $REPO_DIR - git config --global user.email "pandas-docs-bot@localhost.foo" - git config --global user.name "pandas-docs-bot" - git init - touch README - git add README - git commit -m "Initial commit" --allow-empty - git branch gh-pages - git checkout gh-pages - touch .nojekyll - git add --all . - git commit -m "Version" --allow-empty - git remote remove origin - git remote add origin "https://${TOKEN}@github.com/pandas-dev/pandas-docs-travis.git" - git fetch origin - git remote -v - git push origin gh-pages -f - fi - displayName: 'Publishing docs (GitHub pages)' - condition: true - env: - REPO_DIR: $(Build.ArtifactStagingDirectory)/pandas-docs-travis - TOKEN: $(GITHUB_DOCS_TOKEN) - - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev From 455e77cc2b3a3598925d8695f1426389c61d4b6d Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 28 Nov 2018 17:03:52 +0000 Subject: [PATCH 091/104] Adding debug info to diagnose /home/mgarcia/miniconda3/bin:/home/mgarcia/miniconda3/bin:/usr/lib64/qt-3.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/home/mgarcia/.local/bin:/home/mgarcia/bin problem --- azure-pipelines.yml | 2 ++ ci/incremental/setup_conda_environment.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c29b0eba6da80..0e7739c4873b9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -44,7 +44,9 @@ jobs: - script: | sudo apt-get install -y libc6-dev-i386 + echo "PATH before install_miniconda.sh: $PATH" ci/incremental/install_miniconda.sh + echo "PATH before setup_conda_environment.sh: $PATH" ci/incremental/setup_conda_environment.sh displayName: 'Set up environment' diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index f174c17a614d8..eaf07ff17c1d7 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -7,6 +7,7 @@ PIP_INSTALL="pip install -q" # Deactivate any environment +echo "PATH inside setup_conda_environment.sh: $PATH" source deactivate # Display root environment (for debugging) conda list From 7f48ac201878bd18b3b3e3425d20e1dc93e8b5df Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 28 Nov 2018 17:38:35 +0000 Subject: [PATCH 092/104] adding more debug information --- ci/incremental/setup_conda_environment.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index eaf07ff17c1d7..6ef4493957972 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -8,6 +8,7 @@ PIP_INSTALL="pip install -q" # Deactivate any environment echo "PATH inside setup_conda_environment.sh: $PATH" +ls $HOME/miniconda3/bin/ source deactivate # Display root environment (for debugging) conda list From 078a987e9e711d9eb5c2d9b9eef7bba756e50c7f Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 28 Nov 2018 18:00:24 +0000 Subject: [PATCH 093/104] Revert "Removing documentation build from azure" This reverts commit 59d55d87b5564c606e05a9d646a1d15d7f22052c. --- azure-pipelines.yml | 69 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0e7739c4873b9..07bfe0f6a2fdd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -30,7 +30,8 @@ jobs: timeoutInMinutes: 90 steps: - script: | - echo '##vso[task.prependpath]$HOME/miniconda3/bin' + # XXX not sure why setting $PATH here does not work + # echo '##vso[task.setvariable variable=PATH]$HOME/miniconda3/bin:$PATH' echo '##vso[task.setvariable variable=CONDA_ENV]pandas-dev' echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' echo '##vso[task.setvariable variable=AZURE]true' @@ -38,11 +39,13 @@ jobs: # Do not require a conda environment - script: | + export PATH=$HOME/miniconda3/bin:$PATH ci/code_checks.sh patterns displayName: 'Looking for unwanted patterns' condition: true - script: | + export PATH=$HOME/miniconda3/bin:$PATH sudo apt-get install -y libc6-dev-i386 echo "PATH before install_miniconda.sh: $PATH" ci/incremental/install_miniconda.sh @@ -52,18 +55,21 @@ jobs: # Do not require pandas - script: | + export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh lint displayName: 'Linting' condition: true - script: | + export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh dependencies displayName: 'Dependencies consistency' condition: true - script: | + export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/incremental/build.sh displayName: 'Build' @@ -71,29 +77,90 @@ jobs: # Require pandas - script: | + export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh code displayName: 'Checks on imported code' condition: true - script: | + export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh doctests displayName: 'Running doctests' condition: true - script: | + export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh docstrings displayName: 'Docstring validation' condition: true - script: | + export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev pytest --capture=no --strict scripts displayName: 'Testing docstring validaton script' condition: true + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev + doc/make.py html + displayName: 'Building docs' + condition: true + + - script: | + if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then + export NAME="master" + else + export NAME=$(System.PullRequest.PullRequestId) + fi + az extension add --name storage-preview + az storage blob upload-batch --connection-string $CONNECTION_STRING \ + --source $SOURCE \ + --destination $DESTINATION \ + --destination-path $NAME + echo "Documentation uploaded to https://pandas.blob.core.windows.net/docs/$NAME" + displayName: 'Publishing docs (Azure storage)' + condition: true + env: + CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING) + SOURCE: $(Build.SourcesDirectory)/doc/build/html/ + DESTINATION: 'docs' + + - script: | + echo "Publishing documentation to GitHub pages is still done in Travis" + exit 0 + if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then + mkdir -p $REPO_DIR + rm -rf $REPO_DIR/* + cp -r doc/build/html/* $REPO_DIR/ + cd $REPO_DIR + git config --global user.email "pandas-docs-bot@localhost.foo" + git config --global user.name "pandas-docs-bot" + git init + touch README + git add README + git commit -m "Initial commit" --allow-empty + git branch gh-pages + git checkout gh-pages + touch .nojekyll + git add --all . + git commit -m "Version" --allow-empty + git remote remove origin + git remote add origin "https://${TOKEN}@github.com/pandas-dev/pandas-docs-travis.git" + git fetch origin + git remote -v + git push origin gh-pages -f + fi + displayName: 'Publishing docs (GitHub pages)' + condition: true + env: + REPO_DIR: $(Build.ArtifactStagingDirectory)/pandas-docs-travis + TOKEN: $(GITHUB_DOCS_TOKEN) + - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev From d7883a18f71bda43ed0ca5ae8d13a7afa0d1874e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 28 Nov 2018 18:04:26 +0000 Subject: [PATCH 094/104] Clean up of debug commands --- azure-pipelines.yml | 7 +++---- ci/incremental/setup_conda_environment.sh | 2 -- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 07bfe0f6a2fdd..8820a77e8dad9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -30,8 +30,9 @@ jobs: timeoutInMinutes: 90 steps: - script: | - # XXX not sure why setting $PATH here does not work - # echo '##vso[task.setvariable variable=PATH]$HOME/miniconda3/bin:$PATH' + # XXX next command should avoid redefining the path in every step, but + # made the process crash as it couldn't find deactivate + #echo '##vso[task.prependpath]$HOME/miniconda3/bin' echo '##vso[task.setvariable variable=CONDA_ENV]pandas-dev' echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' echo '##vso[task.setvariable variable=AZURE]true' @@ -47,9 +48,7 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH sudo apt-get install -y libc6-dev-i386 - echo "PATH before install_miniconda.sh: $PATH" ci/incremental/install_miniconda.sh - echo "PATH before setup_conda_environment.sh: $PATH" ci/incremental/setup_conda_environment.sh displayName: 'Set up environment' diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index 6ef4493957972..f174c17a614d8 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -7,8 +7,6 @@ PIP_INSTALL="pip install -q" # Deactivate any environment -echo "PATH inside setup_conda_environment.sh: $PATH" -ls $HOME/miniconda3/bin/ source deactivate # Display root environment (for debugging) conda list From c2be491f52f67b3fd7a2dab526fb7e59b95abcb9 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 28 Nov 2018 18:05:47 +0000 Subject: [PATCH 095/104] Removing doc build in azure (restored because of the path problem) --- azure-pipelines.yml | 57 --------------------------------------------- 1 file changed, 57 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8820a77e8dad9..a58f82ec6de49 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -103,63 +103,6 @@ jobs: displayName: 'Testing docstring validaton script' condition: true - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - source activate pandas-dev - doc/make.py html - displayName: 'Building docs' - condition: true - - - script: | - if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then - export NAME="master" - else - export NAME=$(System.PullRequest.PullRequestId) - fi - az extension add --name storage-preview - az storage blob upload-batch --connection-string $CONNECTION_STRING \ - --source $SOURCE \ - --destination $DESTINATION \ - --destination-path $NAME - echo "Documentation uploaded to https://pandas.blob.core.windows.net/docs/$NAME" - displayName: 'Publishing docs (Azure storage)' - condition: true - env: - CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING) - SOURCE: $(Build.SourcesDirectory)/doc/build/html/ - DESTINATION: 'docs' - - - script: | - echo "Publishing documentation to GitHub pages is still done in Travis" - exit 0 - if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then - mkdir -p $REPO_DIR - rm -rf $REPO_DIR/* - cp -r doc/build/html/* $REPO_DIR/ - cd $REPO_DIR - git config --global user.email "pandas-docs-bot@localhost.foo" - git config --global user.name "pandas-docs-bot" - git init - touch README - git add README - git commit -m "Initial commit" --allow-empty - git branch gh-pages - git checkout gh-pages - touch .nojekyll - git add --all . - git commit -m "Version" --allow-empty - git remote remove origin - git remote add origin "https://${TOKEN}@github.com/pandas-dev/pandas-docs-travis.git" - git fetch origin - git remote -v - git push origin gh-pages -f - fi - displayName: 'Publishing docs (GitHub pages)' - condition: true - env: - REPO_DIR: $(Build.ArtifactStagingDirectory)/pandas-docs-travis - TOKEN: $(GITHUB_DOCS_TOKEN) - - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev From 910825a11e40ab04b2ca9e352e29f2ec022df473 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 29 Nov 2018 19:35:28 +0000 Subject: [PATCH 096/104] Building docs in azure, and uploading them to azure storage if it's a commit to master and not a PR --- azure-pipelines.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a58f82ec6de49..9382a498f4d1b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -103,6 +103,29 @@ jobs: displayName: 'Testing docstring validaton script' condition: true + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev + doc/make.py html + displayName: 'Building docs' + condition: true + + - script: | + if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then + az extension add --name storage-preview + az storage blob upload-batch --connection-string $CONNECTION_STRING \ + --source $SOURCE \ + --destination '$web' + echo "Documentation uploaded to https://pandas.blob.core.windows.net" + else + echo "Documentation is only uploaded for commits to master, and not PRs" + fi + displayName: 'Publishing docs (Azure storage)' + condition: true + env: + CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING) + SOURCE: $(Build.SourcesDirectory)/doc/build/html/ + - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev From 4328b04747a15b2fc958c178b11ec988d89b6933 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 1 Dec 2018 01:05:22 +0000 Subject: [PATCH 097/104] Fixing pending docstrings --- pandas/core/window.py | 72 +++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/pandas/core/window.py b/pandas/core/window.py index 6a79d78791c1d..68a36fb2a6999 100644 --- a/pandas/core/window.py +++ b/pandas/core/window.py @@ -30,7 +30,6 @@ _shared_docs = dict(**_shared_docs) _doc_template = """ - Returns ------- same type as input @@ -1340,23 +1339,25 @@ def f(arg, *args, **kwargs): return self._apply(f, 'quantile', quantile=quantile, **kwargs) - _shared_docs['cov'] = dedent(""" - Calculate the %(name)s sample covariance. + _shared_docs['cov'] = """ + Calculate the %(name)s sample covariance. - Parameters - ---------- - other : Series, DataFrame, or ndarray, optional - if not supplied then will default to self and produce pairwise output - pairwise : bool, default None - If False then only matching columns between self and other will be used - and the output will be a DataFrame. - If True then all pairwise combinations will be calculated and the - output will be a MultiIndexed DataFrame in the case of DataFrame - inputs. In the case of missing elements, only complete pairwise - observations will be used. - ddof : int, default 1 - Delta Degrees of Freedom. The divisor used in calculations - is ``N - ddof``, where ``N`` represents the number of elements.""") + Parameters + ---------- + other : Series, DataFrame, or ndarray, optional + If not supplied then will default to self and produce pairwise + output. + pairwise : bool, default None + If False then only matching columns between self and other will be + used and the output will be a DataFrame. + If True then all pairwise combinations will be calculated and the + output will be a MultiIndexed DataFrame in the case of DataFrame + inputs. In the case of missing elements, only complete pairwise + observations will be used. + ddof : int, default 1 + Delta Degrees of Freedom. The divisor used in calculations + is ``N - ddof``, where ``N`` represents the number of elements. + """ def cov(self, other=None, pairwise=None, ddof=1, **kwargs): if other is None: @@ -2054,28 +2055,27 @@ def _constructor(self): _bias_template = """ - -Parameters ----------- -bias : bool, default False - Use a standard estimation bias correction + Parameters + ---------- + bias : bool, default False + Use a standard estimation bias correction """ _pairwise_template = """ - -Parameters ----------- -other : Series, DataFrame, or ndarray, optional - if not supplied then will default to self and produce pairwise output -pairwise : bool, default None - If False then only matching columns between self and other will be used and - the output will be a DataFrame. - If True then all pairwise combinations will be calculated and the output - will be a MultiIndex DataFrame in the case of DataFrame inputs. - In the case of missing elements, only complete pairwise observations will - be used. -bias : bool, default False - Use a standard estimation bias correction + Parameters + ---------- + other : Series, DataFrame, or ndarray, optional + If not supplied then will default to self and produce pairwise + output. + pairwise : bool, default None + If False then only matching columns between self and other will be + used and the output will be a DataFrame. + If True then all pairwise combinations will be calculated and the + output will be a MultiIndex DataFrame in the case of DataFrame + inputs. In the case of missing elements, only complete pairwise + observations will be used. + bias : bool, default False + Use a standard estimation bias correction """ From 4c57f48772e78aaf5e41669082ab7633db790c10 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 1 Dec 2018 01:07:58 +0000 Subject: [PATCH 098/104] Updating dependencies with the version that fixes the speed problems --- environment.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/environment.yml b/environment.yml index 99db4f561c0f3..0571429d06443 100644 --- a/environment.yml +++ b/environment.yml @@ -4,23 +4,22 @@ channels: - conda-forge dependencies: # required - - NumPy + - numpy>=1.15 - python=3 - python-dateutil>=2.5.0 - pytz # development - asv - - Cython>=0.28.2 + - cython>=0.28.2 - flake8 - flake8-comprehensions - flake8-rst=0.4.2 - gitpython - - hypothesis>=3.58.0 + - hypothesis>=3.82 - isort - moto - - pytest>=3.6 - - setuptools>=24.2.0 + - pytest>=4.0 - sphinx - sphinxcontrib-spelling @@ -29,7 +28,6 @@ dependencies: - blosc - bottleneck>=1.2.0 - fastparquet>=0.1.2 - - gcsfs - html5lib - ipython>=5.6.0 - ipykernel @@ -37,15 +35,13 @@ dependencies: - lxml - matplotlib>=2.0.0 - nbsphinx - - numexpr>=2.6.1 + - numexpr>=2.6.8 - openpyxl - pyarrow>=0.7.0 - - pymysql - pytables>=3.4.2 - pytest-cov - pytest-xdist - - s3fs - - scipy>=0.18.1 + - scipy>=1.1 - seaborn - sqlalchemy - statsmodels From 497f0321c0918f6c394b7a46c3afa6ade1e4a19d Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 1 Dec 2018 01:16:32 +0000 Subject: [PATCH 099/104] Updating pip requirements --- requirements-dev.txt | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7d0464b347c8e..3a229ee7a9638 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,24 +1,22 @@ -NumPy +numpy>=1.15 python-dateutil>=2.5.0 pytz asv -Cython>=0.28.2 +cython>=0.28.2 flake8 flake8-comprehensions flake8-rst==0.4.2 gitpython -hypothesis>=3.58.0 +hypothesis>=3.82 isort moto -pytest>=3.6 -setuptools>=24.2.0 +pytest>=4.0 sphinx sphinxcontrib-spelling beautifulsoup4>=4.2.1 blosc bottleneck>=1.2.0 fastparquet>=0.1.2 -gcsfs html5lib ipython>=5.6.0 ipykernel @@ -26,15 +24,13 @@ jinja2 lxml matplotlib>=2.0.0 nbsphinx -numexpr>=2.6.1 +numexpr>=2.6.8 openpyxl pyarrow>=0.7.0 -pymysql tables>=3.4.2 pytest-cov pytest-xdist -s3fs -scipy>=0.18.1 +scipy>=1.1 seaborn sqlalchemy statsmodels From c88911ae29ca8422cfae2278d41683d98711e729 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 1 Dec 2018 01:22:42 +0000 Subject: [PATCH 100/104] Always uploading docs (removed if to test if uploading the docs work in a PR from the same pandas repo) --- azure-pipelines.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9382a498f4d1b..bf8f8185c16c1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -111,15 +111,11 @@ jobs: condition: true - script: | - if [ "$(Build.SourceBranch)" == "refs/heads/master" ]; then - az extension add --name storage-preview - az storage blob upload-batch --connection-string $CONNECTION_STRING \ - --source $SOURCE \ - --destination '$web' - echo "Documentation uploaded to https://pandas.blob.core.windows.net" - else - echo "Documentation is only uploaded for commits to master, and not PRs" - fi + az extension add --name storage-preview + az storage blob upload-batch --connection-string $CONNECTION_STRING \ + --source $SOURCE \ + --destination '$web' + echo "Documentation uploaded to https://pandas.blob.core.windows.net" displayName: 'Publishing docs (Azure storage)' condition: true env: From 75b89eba2067914bf8d1acc97f7318fa6a080c32 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 1 Dec 2018 02:24:32 +0000 Subject: [PATCH 101/104] Adding parallelization to build and docs --- azure-pipelines.yml | 2 +- ci/incremental/build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bf8f8185c16c1..ff79112e71c0d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -106,7 +106,7 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev - doc/make.py html + doc/make.py html --num-jobs=2 displayName: 'Building docs' condition: true diff --git a/ci/incremental/build.sh b/ci/incremental/build.sh index 05648037935a3..7e906530c8f12 100755 --- a/ci/incremental/build.sh +++ b/ci/incremental/build.sh @@ -4,7 +4,7 @@ set -v -e echo "[building extensions]" -python setup.py build_ext -q --inplace +python setup.py build_ext -q --inplace -j2 python -m pip install -e . echo From 011950e05f83dcb052afa1ee275660c20250d478 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 1 Dec 2018 11:12:22 +0000 Subject: [PATCH 102/104] Removing documentation build in azure, and reverting using more than one process in the build --- azure-pipelines.yml | 19 ------------------- ci/incremental/build.sh | 2 +- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ff79112e71c0d..a58f82ec6de49 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -103,25 +103,6 @@ jobs: displayName: 'Testing docstring validaton script' condition: true - - script: | - export PATH=$HOME/miniconda3/bin:$PATH - source activate pandas-dev - doc/make.py html --num-jobs=2 - displayName: 'Building docs' - condition: true - - - script: | - az extension add --name storage-preview - az storage blob upload-batch --connection-string $CONNECTION_STRING \ - --source $SOURCE \ - --destination '$web' - echo "Documentation uploaded to https://pandas.blob.core.windows.net" - displayName: 'Publishing docs (Azure storage)' - condition: true - env: - CONNECTION_STRING: $(AZURE_STORAGE_CONNECTION_STRING) - SOURCE: $(Build.SourcesDirectory)/doc/build/html/ - - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev diff --git a/ci/incremental/build.sh b/ci/incremental/build.sh index 7e906530c8f12..05648037935a3 100755 --- a/ci/incremental/build.sh +++ b/ci/incremental/build.sh @@ -4,7 +4,7 @@ set -v -e echo "[building extensions]" -python setup.py build_ext -q --inplace -j2 +python setup.py build_ext -q --inplace python -m pip install -e . echo From 01942b93548428866d8adc1c91eaca63ae1e2fcb Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 1 Dec 2018 11:19:02 +0000 Subject: [PATCH 103/104] Made an invgrep a newly added pattern validation --- ci/code_checks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 6a8d34f41e703..a8a86eedb0549 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -145,7 +145,7 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then RET=$(($RET + $?)) ; echo $MSG "DONE" MSG='Check that the deprecated `assert_raises_regex` is not used (`pytest.raises(match=pattern)` should be used instead)' ; echo $MSG - ! grep -R --exclude=*.pyc --exclude=testing.py --exclude=test_testing.py assert_raises_regex pandas + invgrep -R --exclude=*.pyc --exclude=testing.py --exclude=test_testing.py assert_raises_regex pandas RET=$(($RET + $?)) ; echo $MSG "DONE" fi From 705eb9dbf8e872164a9461a1100b807aa3de7f4d Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 1 Dec 2018 12:14:49 +0000 Subject: [PATCH 104/104] Regenerating pip dependencies --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1dbfb7fc04774..3a229ee7a9638 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -38,4 +38,4 @@ xarray xlrd xlsxwriter xlwt -cpplint +cpplint \ No newline at end of file