From b78b4ce5c561c76609531be39ab4f84287fbab7f Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 18 Jun 2019 21:46:06 +0100 Subject: [PATCH 1/7] CI: Running tests with xfvb for the clipboard to work --- ci/azure/posix.yml | 5 +++++ ci/run_tests.sh | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index c5676e0a2a6a0..62532f7630318 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -53,18 +53,22 @@ jobs: echo "Creating Environment" ci/setup_env.sh displayName: 'Setup environment and build pandas' + - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/run_tests.sh displayName: 'Test' + - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd + - task: PublishTestResults@2 inputs: testResultsFiles: 'test-data-*.xml' testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }} + - powershell: | $junitXml = "test-data-single.xml" $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"' @@ -90,6 +94,7 @@ jobs: Write-Error "$($matches[1]) tests failed" } displayName: 'Check for test failures' + - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev diff --git a/ci/run_tests.sh b/ci/run_tests.sh index ee46da9f52eab..592dcbbbfd577 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -46,6 +46,11 @@ do fi PYTEST_CMD="pytest -m \"$TYPE_PATTERN$PATTERN\" -n $NUM_JOBS -s --strict --durations=10 --junitxml=test-data-$TYPE.xml $TEST_ARGS $COVERAGE pandas" + if [[ "$(uname)" == "Linux" ]]; then + # This is required for the clipboard to work (X virtual frambuffer implements the X server API, required by the clipboard tools, without running an actual X server) + DISPLAY=:99.0 + PYTEST_CMD="xvfb-run $PYTEST_CMD" + fi echo $PYTEST_CMD # if no tests are found (the case of "single and slow"), pytest exits with code 5, and would make the script fail, if not for the below code sh -c "$PYTEST_CMD; ret=\$?; [ \$ret = 5 ] && exit 0 || exit \$ret" From 05d5444691ca6006c0e75988406661d773f633c9 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 19 Jun 2019 10:04:07 +0100 Subject: [PATCH 2/7] Showing list of running processes before running the tests, to see if xvfb is still running, as it looks like --- ci/run_tests.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 592dcbbbfd577..91066a312a918 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -51,6 +51,11 @@ do DISPLAY=:99.0 PYTEST_CMD="xvfb-run $PYTEST_CMD" fi + echo "################################################################################" + echo "ps -ef" + echo "################################################################################" + ps -ef + echo "################################################################################" echo $PYTEST_CMD # if no tests are found (the case of "single and slow"), pytest exits with code 5, and would make the script fail, if not for the below code sh -c "$PYTEST_CMD; ret=\$?; [ \$ret = 5 ] && exit 0 || exit \$ret" From 004d2d06c1bfe11ac1a561d9492b5a819bbd9d98 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 19 Jun 2019 10:49:40 +0100 Subject: [PATCH 3/7] Removing travis xvfb (now in run_tests.sh) --- .travis.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8335a6ee92bef..03a7a486ba0ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,14 +79,6 @@ install: - ci/submit_cython_cache.sh - echo "install done" -before_script: - # display server (for clipboard functionality) needs to be started here, - # does not work if done in install:setup_env.sh (GH-26103) - - export DISPLAY=":99.0" - - echo "sh -e /etc/init.d/xvfb start" - - sh -e /etc/init.d/xvfb start - - sleep 3 - script: - echo "script start" - source activate pandas-dev From 3bba0d7f21c69e5d3a4364572618455fb2f7df16 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 19 Jun 2019 13:20:18 +0100 Subject: [PATCH 4/7] Fixing xvfb with a sleep and not running in travis --- ci/run_tests.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 91066a312a918..a42c06bc5b249 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -46,16 +46,15 @@ do fi PYTEST_CMD="pytest -m \"$TYPE_PATTERN$PATTERN\" -n $NUM_JOBS -s --strict --durations=10 --junitxml=test-data-$TYPE.xml $TEST_ARGS $COVERAGE pandas" - if [[ "$(uname)" == "Linux" ]]; then + if [[ "$(uname)" == "Linux" && "STRAVIS" != "true" ]]; then # This is required for the clipboard to work (X virtual frambuffer implements the X server API, required by the clipboard tools, without running an actual X server) DISPLAY=:99.0 PYTEST_CMD="xvfb-run $PYTEST_CMD" + # Sleeping a bit the second time we execute the tests with xvfb to make sure the previous execution finished + if [[ "$TYPE_PATTERN" == "multiple" ]]; then + sleep 3 + fi fi - echo "################################################################################" - echo "ps -ef" - echo "################################################################################" - ps -ef - echo "################################################################################" echo $PYTEST_CMD # if no tests are found (the case of "single and slow"), pytest exits with code 5, and would make the script fail, if not for the below code sh -c "$PYTEST_CMD; ret=\$?; [ \$ret = 5 ] && exit 0 || exit \$ret" From d3653d5afa139336a3cbfdfd66793538a2db46aa Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 19 Jun 2019 13:25:39 +0100 Subject: [PATCH 5/7] Restoring travis xvfb stuff --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index 03a7a486ba0ce..8335a6ee92bef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,6 +79,14 @@ install: - ci/submit_cython_cache.sh - echo "install done" +before_script: + # display server (for clipboard functionality) needs to be started here, + # does not work if done in install:setup_env.sh (GH-26103) + - export DISPLAY=":99.0" + - echo "sh -e /etc/init.d/xvfb start" + - sh -e /etc/init.d/xvfb start + - sleep 3 + script: - echo "script start" - source activate pandas-dev From f9804fc4d97d2c1b8578d1261768d772c81d02ae Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 19 Jun 2019 16:49:47 +0100 Subject: [PATCH 6/7] Adding debug info --- ci/run_tests.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index a42c06bc5b249..180963cf6c1f3 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -46,6 +46,10 @@ do fi PYTEST_CMD="pytest -m \"$TYPE_PATTERN$PATTERN\" -n $NUM_JOBS -s --strict --durations=10 --junitxml=test-data-$TYPE.xml $TEST_ARGS $COVERAGE pandas" + echo "uname" + uname + echo "env" + env if [[ "$(uname)" == "Linux" && "STRAVIS" != "true" ]]; then # This is required for the clipboard to work (X virtual frambuffer implements the X server API, required by the clipboard tools, without running an actual X server) DISPLAY=:99.0 From 3a6c641bca44ebc6445940018e1d740ac8462c81 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 19 Jun 2019 17:35:01 +0100 Subject: [PATCH 7/7] Changing the condition to run only in Linux/pipelines, and the way we wait for previous run to finish --- ci/run_tests.sh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 180963cf6c1f3..103385c9a2541 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -46,22 +46,20 @@ do fi PYTEST_CMD="pytest -m \"$TYPE_PATTERN$PATTERN\" -n $NUM_JOBS -s --strict --durations=10 --junitxml=test-data-$TYPE.xml $TEST_ARGS $COVERAGE pandas" - echo "uname" - uname - echo "env" - env - if [[ "$(uname)" == "Linux" && "STRAVIS" != "true" ]]; then - # This is required for the clipboard to work (X virtual frambuffer implements the X server API, required by the clipboard tools, without running an actual X server) + echo $AGENT_OS + if [[ "$AGENT_OS" == "Linux" ]]; then + echo "Emulating X server with xvfb for clipboard tests to run (this should only be displayed in Linux build of Azure Pipelines)" DISPLAY=:99.0 PYTEST_CMD="xvfb-run $PYTEST_CMD" # Sleeping a bit the second time we execute the tests with xvfb to make sure the previous execution finished - if [[ "$TYPE_PATTERN" == "multiple" ]]; then - sleep 3 + if [ -n $PYTEST_PID ]; then + timeout 60 tail --pid=$pid -f /dev/null fi fi echo $PYTEST_CMD # if no tests are found (the case of "single and slow"), pytest exits with code 5, and would make the script fail, if not for the below code sh -c "$PYTEST_CMD; ret=\$?; [ \$ret = 5 ] && exit 0 || exit \$ret" + PYTEST_PID=$! if [[ "$COVERAGE" && $? == 0 ]]; then echo "uploading coverage for $TYPE tests"