From ca3898a7184ce379f0e7c8779a65dbb1c3034f0e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 19 Nov 2019 16:36:46 +0000 Subject: [PATCH 01/23] CI: Fix inconsistent clipboard test --- ci/run_tests.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index b91cfb3bed8cc..4cd7a465ca1cd 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -29,8 +29,7 @@ PYTEST_CMD="pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --duratio # Travis does not have have an X server if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - DISPLAY=DISPLAY=:99.0 - PYTEST_CMD="xvfb-run -e /dev/stdout $PYTEST_CMD" + PYTEST_CMD="xvfb-run -a -e /dev/stdout $PYTEST_CMD" fi echo $PYTEST_CMD From d72987f99042854ac6f27f2211e4b85a2617c8d8 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 19 Nov 2019 10:13:07 -0600 Subject: [PATCH 02/23] lazy check (cherry picked from commit 6557e9b10f4e4138f674bff1788a56d109c66617) --- pandas/tests/io/test_clipboard.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pandas/tests/io/test_clipboard.py b/pandas/tests/io/test_clipboard.py index 4559ba264d8b7..2801a83384278 100644 --- a/pandas/tests/io/test_clipboard.py +++ b/pandas/tests/io/test_clipboard.py @@ -10,12 +10,6 @@ from pandas.io.clipboard import PyperclipException, clipboard_get, clipboard_set -try: - DataFrame({"A": [1, 2]}).to_clipboard() - _DEPS_INSTALLED = 1 -except (PyperclipException, RuntimeError): - _DEPS_INSTALLED = 0 - def build_kwargs(sep, excel): kwargs = {} @@ -146,9 +140,17 @@ def test_mock_clipboard(mock_clipboard): assert result == "abc" +@pytest.fixture(scope="module") +def check_clipboard(): + """Check if we have a clipboard, skipping if it's not present""" + try: + DataFrame({"A": [1, 2]}).to_clipboard() + except (PyperclipException, RuntimeError): + pytest.skip("Missing clipboard dependency") + + @pytest.mark.single @pytest.mark.clipboard -@pytest.mark.skipif(not _DEPS_INSTALLED, reason="clipboard primitives not installed") @pytest.mark.usefixtures("mock_clipboard") class TestClipboard: def check_round_trip_frame(self, data, excel=None, sep=None, encoding=None): @@ -256,8 +258,8 @@ def test_round_trip_valid_encodings(self, enc, df): @pytest.mark.single @pytest.mark.clipboard -@pytest.mark.skipif(not _DEPS_INSTALLED, reason="clipboard primitives not installed") @pytest.mark.parametrize("data", ["\U0001f44d...", "Ωœ∑´...", "abcd..."]) +@pytest.mark.usefixtures("check_clipboard") def test_raw_roundtrip(data): # PR #25040 wide unicode wasn't copied correctly on PY3 on windows clipboard_set(data) From 5e0e32c5551a2fcbdf187d0684da235cd79e9149 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 20 Nov 2019 01:22:12 +0000 Subject: [PATCH 03/23] Running clipboard tests in all builds (I think we already do in practice), and moving unicode test with the others --- pandas/tests/io/test_clipboard.py | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/pandas/tests/io/test_clipboard.py b/pandas/tests/io/test_clipboard.py index 2801a83384278..4e5a4fb7dadbf 100644 --- a/pandas/tests/io/test_clipboard.py +++ b/pandas/tests/io/test_clipboard.py @@ -140,15 +140,6 @@ def test_mock_clipboard(mock_clipboard): assert result == "abc" -@pytest.fixture(scope="module") -def check_clipboard(): - """Check if we have a clipboard, skipping if it's not present""" - try: - DataFrame({"A": [1, 2]}).to_clipboard() - except (PyperclipException, RuntimeError): - pytest.skip("Missing clipboard dependency") - - @pytest.mark.single @pytest.mark.clipboard @pytest.mark.usefixtures("mock_clipboard") @@ -255,12 +246,8 @@ def test_invalid_encoding(self, df): def test_round_trip_valid_encodings(self, enc, df): self.check_round_trip_frame(df, encoding=enc) - -@pytest.mark.single -@pytest.mark.clipboard -@pytest.mark.parametrize("data", ["\U0001f44d...", "Ωœ∑´...", "abcd..."]) -@pytest.mark.usefixtures("check_clipboard") -def test_raw_roundtrip(data): - # PR #25040 wide unicode wasn't copied correctly on PY3 on windows - clipboard_set(data) - assert data == clipboard_get() + @pytest.mark.parametrize("data", ["\U0001f44d...", "Ωœ∑´...", "abcd..."]) + def test_raw_roundtrip(data): + # PR #25040 wide unicode wasn't copied correctly on PY3 on windows + clipboard_set(data) + assert data == clipboard_get() From 5fe92091aa5655a07d45c8713aed8adfc8b931f8 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 20 Nov 2019 01:26:45 +0000 Subject: [PATCH 04/23] Adding missing self --- pandas/tests/io/test_clipboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_clipboard.py b/pandas/tests/io/test_clipboard.py index 4e5a4fb7dadbf..80c73799788af 100644 --- a/pandas/tests/io/test_clipboard.py +++ b/pandas/tests/io/test_clipboard.py @@ -247,7 +247,7 @@ def test_round_trip_valid_encodings(self, enc, df): self.check_round_trip_frame(df, encoding=enc) @pytest.mark.parametrize("data", ["\U0001f44d...", "Ωœ∑´...", "abcd..."]) - def test_raw_roundtrip(data): + def test_raw_roundtrip(self, data): # PR #25040 wide unicode wasn't copied correctly on PY3 on windows clipboard_set(data) assert data == clipboard_get() From 99315bf85bea8df8b2115f15d3ace9776a0527d0 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 20 Nov 2019 01:52:23 +0000 Subject: [PATCH 05/23] Restoring test outside class --- pandas/tests/io/test_clipboard.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pandas/tests/io/test_clipboard.py b/pandas/tests/io/test_clipboard.py index 80c73799788af..0feaf493c2294 100644 --- a/pandas/tests/io/test_clipboard.py +++ b/pandas/tests/io/test_clipboard.py @@ -246,8 +246,10 @@ def test_invalid_encoding(self, df): def test_round_trip_valid_encodings(self, enc, df): self.check_round_trip_frame(df, encoding=enc) - @pytest.mark.parametrize("data", ["\U0001f44d...", "Ωœ∑´...", "abcd..."]) - def test_raw_roundtrip(self, data): - # PR #25040 wide unicode wasn't copied correctly on PY3 on windows - clipboard_set(data) - assert data == clipboard_get() +@pytest.mark.single +@pytest.mark.clipboard +@pytest.mark.parametrize("data", ["\U0001f44d...", "Ωœ∑´...", "abcd..."]) +def test_raw_roundtrip(data): + # PR #25040 wide unicode wasn't copied correctly on PY3 on windows + clipboard_set(data) + assert data == clipboard_get() From 1e53de66b379983a9c17ae613af61dde19041ebb Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 20 Nov 2019 02:35:41 +0000 Subject: [PATCH 06/23] Explicitly skipping clipboard tests in the travis config (azure is down, not sure where they run) --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0acd386eea9ed..e0ce3497a7d6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,11 +32,11 @@ matrix: include: - dist: trusty env: - - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network)" + - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network and not clipboard)" - dist: trusty env: - - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)" + - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network and not clipboard)" - dist: trusty env: From 7c527c8becd43067f770713e75881d37153087f5 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 20 Nov 2019 10:26:59 +0000 Subject: [PATCH 07/23] Disabling clipboard where it should not be tested, installing xsel where it should, and fixing linting --- ci/azure/posix.yml | 9 ++++++--- pandas/tests/io/test_clipboard.py | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 66960ca2c6c10..72065665f2ab5 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -18,24 +18,26 @@ jobs: py36_minimum_versions: ENV_FILE: ci/deps/azure-36-minimum_versions.yaml CONDA_PY: "36" - PATTERN: "not slow and not network" + PATTERN: "not slow and not network and not clipboard" + py36_locale_slow_old_np: ENV_FILE: ci/deps/azure-36-locale.yaml CONDA_PY: "36" PATTERN: "slow" LOCALE_OVERRIDE: "zh_CN.UTF-8" - EXTRA_APT: "language-pack-zh-hans" + EXTRA_APT: "language-pack-zh-hans xsel" py36_locale_slow: ENV_FILE: ci/deps/azure-36-locale_slow.yaml CONDA_PY: "36" PATTERN: "not slow and not network" LOCALE_OVERRIDE: "it_IT.UTF-8" + EXTRA_APT: "xsel" py36_32bit: ENV_FILE: ci/deps/azure-36-32bit.yaml CONDA_PY: "36" - PATTERN: "not slow and not network" + PATTERN: "not slow and not network and not clipboard" BITS32: "yes" py37_locale: @@ -43,6 +45,7 @@ jobs: CONDA_PY: "37" PATTERN: "not slow and not network" LOCALE_OVERRIDE: "zh_CN.UTF-8" + EXTRA_APT: "xsel" # https://github.com/pandas-dev/pandas/issues/29432 # py37_np_dev: diff --git a/pandas/tests/io/test_clipboard.py b/pandas/tests/io/test_clipboard.py index 0feaf493c2294..513cacfe15d09 100644 --- a/pandas/tests/io/test_clipboard.py +++ b/pandas/tests/io/test_clipboard.py @@ -8,7 +8,7 @@ from pandas import DataFrame, get_option, read_clipboard import pandas.util.testing as tm -from pandas.io.clipboard import PyperclipException, clipboard_get, clipboard_set +from pandas.io.clipboard import clipboard_get, clipboard_set def build_kwargs(sep, excel): @@ -246,6 +246,7 @@ def test_invalid_encoding(self, df): def test_round_trip_valid_encodings(self, enc, df): self.check_round_trip_frame(df, encoding=enc) + @pytest.mark.single @pytest.mark.clipboard @pytest.mark.parametrize("data", ["\U0001f44d...", "Ωœ∑´...", "abcd..."]) From 36ff3cea9c40f41e5261cfb1531c4dd3cb526fb2 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 20 Nov 2019 10:56:05 +0000 Subject: [PATCH 08/23] Removing qt from conda --- ci/setup_env.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 3d79c0cfd7000..62ad398cf6896 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -114,6 +114,11 @@ echo "remove postgres if has been installed with conda" echo "we use the one from the CI" conda remove postgresql -y --force || true +echo +echo "remove qt" +echo "causes problems with the clipboard, we use xsel for that" +conda remove qt -y --force || true + echo echo "conda list pandas" conda list pandas From 63c73644725311f0e9b2facdb7e67ee518f48a2a Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 20 Nov 2019 12:21:50 +0000 Subject: [PATCH 09/23] Setting DISPLAY env var --- ci/run_tests.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 4cd7a465ca1cd..3fa8308481ffc 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -25,11 +25,14 @@ if [ "$COVERAGE" ]; then COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME" fi +# An X server has to exit, and DISPLAY set, for the clipboard (and its tests) to work +export DISPLAY=":99.0" + PYTEST_CMD="pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" # Travis does not have have an X server if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - PYTEST_CMD="xvfb-run -a -e /dev/stdout $PYTEST_CMD" + PYTEST_CMD="xvfb-run -e /dev/stdout $PYTEST_CMD" fi echo $PYTEST_CMD From ebfc48bd59ff5dc9a2ef01896b8be54380dd0922 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 20 Nov 2019 12:52:03 +0000 Subject: [PATCH 10/23] Trying to understand if xsel is found or not --- ci/run_tests.sh | 3 +++ pandas/io/clipboard/__init__.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 3fa8308481ffc..e74d0ee828117 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -38,6 +38,9 @@ fi echo $PYTEST_CMD sh -c "$PYTEST_CMD" +echo "Is xsel installed?" +python -c "import subprocess; print(subprocess.call(['which', 'xsel'], stdout=subprocess.PIPE, stderr=subprocess.PIPE))" + if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then echo "uploading coverage" echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME" diff --git a/pandas/io/clipboard/__init__.py b/pandas/io/clipboard/__init__.py index 4f690a57893d1..80959cfc9ffbe 100644 --- a/pandas/io/clipboard/__init__.py +++ b/pandas/io/clipboard/__init__.py @@ -542,6 +542,9 @@ def determine_clipboard(): if HAS_DISPLAY: if _executable_exists("xsel"): return init_xsel_clipboard() + + raise ValueError('xsel is not found') + if _executable_exists("xclip"): return init_xclip_clipboard() if _executable_exists("klipper") and _executable_exists("qdbus"): @@ -568,6 +571,8 @@ def determine_clipboard(): else: return init_qt_clipboard() + raise ValueError('DISPLAY is not set') + return init_no_clipboard() From f84ac34d1e0c0f1f1204067e7d32062b518f1b9c Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 20 Nov 2019 17:15:43 +0000 Subject: [PATCH 11/23] Better checking to see if xsel is installed --- ci/run_tests.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index e74d0ee828117..68e57519a7408 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -28,19 +28,21 @@ fi # An X server has to exit, and DISPLAY set, for the clipboard (and its tests) to work export DISPLAY=":99.0" -PYTEST_CMD="pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" +PYTEST_CMD="pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas/tests/io/test_clipboard.py" # Travis does not have have an X server if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then PYTEST_CMD="xvfb-run -e /dev/stdout $PYTEST_CMD" fi -echo $PYTEST_CMD -sh -c "$PYTEST_CMD" - +set -x echo "Is xsel installed?" +which xsel || true python -c "import subprocess; print(subprocess.call(['which', 'xsel'], stdout=subprocess.PIPE, stderr=subprocess.PIPE))" +echo $PYTEST_CMD +sh -c "$PYTEST_CMD" + if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then echo "uploading coverage" echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME" From c15f5a7e9e8f1d8503b011e79dba926974f24db9 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 20 Nov 2019 20:30:46 +0000 Subject: [PATCH 12/23] More xsel info --- ci/run_tests.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 68e57519a7408..c30af73a567ff 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -35,10 +35,12 @@ if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then PYTEST_CMD="xvfb-run -e /dev/stdout $PYTEST_CMD" fi -set -x echo "Is xsel installed?" -which xsel || true +set -x +dpkg -l | grep xsel +echo $PATH python -c "import subprocess; print(subprocess.call(['which', 'xsel'], stdout=subprocess.PIPE, stderr=subprocess.PIPE))" +which xsel echo $PYTEST_CMD sh -c "$PYTEST_CMD" From 85325551e6092152abbb2963bbb4b7394c0b0110 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 30 Dec 2019 22:31:00 +0000 Subject: [PATCH 13/23] Removing xfail --- pandas/tests/io/test_clipboard.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/io/test_clipboard.py b/pandas/tests/io/test_clipboard.py index b17b0d5fbd627..513cacfe15d09 100644 --- a/pandas/tests/io/test_clipboard.py +++ b/pandas/tests/io/test_clipboard.py @@ -250,7 +250,6 @@ def test_round_trip_valid_encodings(self, enc, df): @pytest.mark.single @pytest.mark.clipboard @pytest.mark.parametrize("data", ["\U0001f44d...", "Ωœ∑´...", "abcd..."]) -@pytest.mark.xfail(reason="flaky in CI", strict=False) def test_raw_roundtrip(data): # PR #25040 wide unicode wasn't copied correctly on PY3 on windows clipboard_set(data) From ed429ed0897436918eb914d1c88a193ccb02ef41 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 31 Dec 2019 00:34:44 +0000 Subject: [PATCH 14/23] Fixing debug info --- ci/run_tests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 28a357d27c77c..89e0a7b2e8eda 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -35,12 +35,14 @@ if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then PYTEST_CMD="xvfb-run -e /dev/stdout $PYTEST_CMD" fi +set +e echo "Is xsel installed?" set -x -dpkg -l | grep xsel +dpkg -l echo $PATH python -c "import subprocess; print(subprocess.call(['which', 'xsel'], stdout=subprocess.PIPE, stderr=subprocess.PIPE))" which xsel +set -e echo $PYTEST_CMD sh -c "$PYTEST_CMD" From fec13636533be548079ac0ce5bf64a893003ba72 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 8 Jan 2020 13:31:04 +0000 Subject: [PATCH 15/23] Black --- pandas/io/clipboard/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/io/clipboard/__init__.py b/pandas/io/clipboard/__init__.py index 24462951379be..be7e99d14af54 100644 --- a/pandas/io/clipboard/__init__.py +++ b/pandas/io/clipboard/__init__.py @@ -541,7 +541,7 @@ def determine_clipboard(): if _executable_exists("xsel"): return init_xsel_clipboard() - raise ValueError('xsel is not found') + raise ValueError("xsel is not found") if _executable_exists("xclip"): return init_xclip_clipboard() @@ -569,7 +569,7 @@ def determine_clipboard(): else: return init_qt_clipboard() - raise ValueError('DISPLAY is not set') + raise ValueError("DISPLAY is not set") return init_no_clipboard() From 7cff22d666a0c2eb44a35f14ea7657f59dbdbca5 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 10 Jan 2020 16:22:05 +0000 Subject: [PATCH 16/23] Fixed typo in a comment --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index c33e2549f7409..d904861fa36b8 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -14,7 +14,7 @@ if [ "$COVERAGE" ]; then COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME" fi -# An X server has to exit, and DISPLAY set, for the clipboard (and its tests) to work +# An X server has to exist, and DISPLAY set, for the clipboard (and its tests) to work export DISPLAY=":99.0" PYTEST_CMD="pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas/tests/io/test_clipboard.py" From 49181e32c136e47a037c0b3a922e559bdba342e4 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 14 Jan 2020 23:25:21 +0000 Subject: [PATCH 17/23] Checking xsel --- ci/run_tests.sh | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index d904861fa36b8..88ba5c31b56cc 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -14,8 +14,13 @@ if [ "$COVERAGE" ]; then COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME" fi -# An X server has to exist, and DISPLAY set, for the clipboard (and its tests) to work -export DISPLAY=":99.0" +if [[ "not clipboard" != *"$PATTERN"* ]]; then + # An X server has to exist, and DISPLAY set, for the clipboard (and its tests) to work + export DISPLAY=":0" + echo "xsel path: $(which xsel)" + python -c "import pandas.io.clipboard ; pandas.io.clipboard.clipboard_set('123')" + python -c "import pandas.io.clipboard ; print(f'clipboard content (should be 123): {pandas.io.clipboard.clipboard_get()}')" +fi PYTEST_CMD="pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas/tests/io/test_clipboard.py" @@ -24,15 +29,6 @@ if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then PYTEST_CMD="xvfb-run -e /dev/stdout $PYTEST_CMD" fi -set +e -echo "Is xsel installed?" -set -x -dpkg -l -echo $PATH -python -c "import subprocess; print(subprocess.call(['which', 'xsel'], stdout=subprocess.PIPE, stderr=subprocess.PIPE))" -which xsel -set -e - echo $PYTEST_CMD sh -c "$PYTEST_CMD" From 8a8244d8b5dab646ad1297ea9efa580ec86ea13a Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 14 Jan 2020 23:39:30 +0000 Subject: [PATCH 18/23] Adding more checks for xsel --- ci/run_tests.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 88ba5c31b56cc..0d519f6ac9d48 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -18,6 +18,11 @@ if [[ "not clipboard" != *"$PATTERN"* ]]; then # An X server has to exist, and DISPLAY set, for the clipboard (and its tests) to work export DISPLAY=":0" echo "xsel path: $(which xsel)" + echo "testing xsel from terminal" + echo "DISPLAY: $DISPLAY" + echo "abc" | xsel -i + echo "clipboard content (should be abc): $(xsel -o)" + echo "testing xsel from pandas" python -c "import pandas.io.clipboard ; pandas.io.clipboard.clipboard_set('123')" python -c "import pandas.io.clipboard ; print(f'clipboard content (should be 123): {pandas.io.clipboard.clipboard_get()}')" fi From 3384d1ebec59d9466ed4fd7d6016f4a6558bb19e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 15 Jan 2020 00:06:49 +0000 Subject: [PATCH 19/23] Using xvfb --- ci/run_tests.sh | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 0d519f6ac9d48..9db4222ed2331 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -15,25 +15,26 @@ if [ "$COVERAGE" ]; then fi if [[ "not clipboard" != *"$PATTERN"* ]]; then + echo "DISPLAY (original): $DISPLAY" + # An X server has to exist, and DISPLAY set, for the clipboard (and its tests) to work - export DISPLAY=":0" + if [ -z $DISPLAY ]; then + export DISPLAY=":0" + XVFB="xvfb-run -e /dev/stdout " + fi + echo "xsel path: $(which xsel)" - echo "testing xsel from terminal" echo "DISPLAY: $DISPLAY" - echo "abc" | xsel -i - echo "clipboard content (should be abc): $(xsel -o)" - echo "testing xsel from pandas" - python -c "import pandas.io.clipboard ; pandas.io.clipboard.clipboard_set('123')" - python -c "import pandas.io.clipboard ; print(f'clipboard content (should be 123): {pandas.io.clipboard.clipboard_get()}')" -fi -PYTEST_CMD="pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas/tests/io/test_clipboard.py" + echo "testing xsel from terminal" + xvfb-run -e /dev/stdout "echo \"terminal clipboard works\" | xsel -i ; echo \"clipboard content: $(xsel -o)\"" -# Travis does not have have an X server -if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - PYTEST_CMD="xvfb-run -e /dev/stdout $PYTEST_CMD" + echo "testing xsel from pandas" + python -c "import pandas.io.clipboard ; pandas.io.clipboard.clipboard_set('pandas clipboard works') ; print(f'clipboard content: {pandas.io.clipboard.clipboard_get()}')" fi +PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas/tests/io/test_clipboard.py" + echo $PYTEST_CMD sh -c "$PYTEST_CMD" From 670409e9d3813eb530e59066ef500744cee1c4a2 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 15 Jan 2020 00:29:12 +0000 Subject: [PATCH 20/23] Not logging xvfb output --- ci/run_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 9db4222ed2331..fde430312d912 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -18,16 +18,16 @@ if [[ "not clipboard" != *"$PATTERN"* ]]; then echo "DISPLAY (original): $DISPLAY" # An X server has to exist, and DISPLAY set, for the clipboard (and its tests) to work - if [ -z $DISPLAY ]; then + if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then export DISPLAY=":0" - XVFB="xvfb-run -e /dev/stdout " + XVFB="xvfb-run " fi echo "xsel path: $(which xsel)" echo "DISPLAY: $DISPLAY" echo "testing xsel from terminal" - xvfb-run -e /dev/stdout "echo \"terminal clipboard works\" | xsel -i ; echo \"clipboard content: $(xsel -o)\"" + xvfb-run "echo \"terminal clipboard works\" | xsel -i ; echo \"clipboard content: $(xsel -o)\"" echo "testing xsel from pandas" python -c "import pandas.io.clipboard ; pandas.io.clipboard.clipboard_set('pandas clipboard works') ; print(f'clipboard content: {pandas.io.clipboard.clipboard_get()}')" From 4a4e116e1a102c7e7fe7c2f84197afcb5aa3293c Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 15 Jan 2020 00:44:10 +0000 Subject: [PATCH 21/23] Removing checks --- ci/run_tests.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index fde430312d912..847f0b4b65c6a 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -16,21 +16,10 @@ fi if [[ "not clipboard" != *"$PATTERN"* ]]; then echo "DISPLAY (original): $DISPLAY" - - # An X server has to exist, and DISPLAY set, for the clipboard (and its tests) to work if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then export DISPLAY=":0" XVFB="xvfb-run " fi - - echo "xsel path: $(which xsel)" - echo "DISPLAY: $DISPLAY" - - echo "testing xsel from terminal" - xvfb-run "echo \"terminal clipboard works\" | xsel -i ; echo \"clipboard content: $(xsel -o)\"" - - echo "testing xsel from pandas" - python -c "import pandas.io.clipboard ; pandas.io.clipboard.clipboard_set('pandas clipboard works') ; print(f'clipboard content: {pandas.io.clipboard.clipboard_get()}')" fi PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas/tests/io/test_clipboard.py" From 2ec47e56f52800092190deec70ecd1c55aea890d Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 15 Jan 2020 01:06:59 +0000 Subject: [PATCH 22/23] Fixing patterns, calling all tests (just debug raises left to be merged) --- .travis.yml | 4 ++-- ci/run_tests.sh | 12 +++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38a85ff695860..b5897e3526327 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,13 +37,13 @@ matrix: - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network and not clipboard)" - env: - - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1" + - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1" services: - mysql - postgresql - env: - - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1" + - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1" services: - mysql - postgresql diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 847f0b4b65c6a..0cb1f4aabf352 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -14,15 +14,13 @@ if [ "$COVERAGE" ]; then COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME" fi -if [[ "not clipboard" != *"$PATTERN"* ]]; then - echo "DISPLAY (original): $DISPLAY" - if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then - export DISPLAY=":0" - XVFB="xvfb-run " - fi +# If no X server is found, we use xvfb to emulate it +if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then + export DISPLAY=":0" + XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas/tests/io/test_clipboard.py" +PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" echo $PYTEST_CMD sh -c "$PYTEST_CMD" From 4a9ed03752135365d904b9f8229ec42b136cfb0e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 15 Jan 2020 01:42:16 +0000 Subject: [PATCH 23/23] Final clean up, this should be ready --- ci/azure/posix.yml | 2 +- pandas/io/clipboard/__init__.py | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index b898d5ebf8518..c9a2e4eefd19d 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -28,7 +28,7 @@ jobs: # we should test with encodings different than utf8, but doesn't seem like Ubuntu supports any LANG: "zh_CN.utf8" LC_ALL: "zh_CN.utf8" - EXTRA_APT: "language-pack-zh-hans xsel" + EXTRA_APT: "language-pack-zh-hans" py36_locale: ENV_FILE: ci/deps/azure-36-locale.yaml diff --git a/pandas/io/clipboard/__init__.py b/pandas/io/clipboard/__init__.py index be7e99d14af54..f808b7e706afb 100644 --- a/pandas/io/clipboard/__init__.py +++ b/pandas/io/clipboard/__init__.py @@ -540,9 +540,6 @@ def determine_clipboard(): if HAS_DISPLAY: if _executable_exists("xsel"): return init_xsel_clipboard() - - raise ValueError("xsel is not found") - if _executable_exists("xclip"): return init_xclip_clipboard() if _executable_exists("klipper") and _executable_exists("qdbus"): @@ -569,8 +566,6 @@ def determine_clipboard(): else: return init_qt_clipboard() - raise ValueError("DISPLAY is not set") - return init_no_clipboard()