diff --git a/.travis.yml b/.travis.yml
index c24c6f06de1fc..a23bc8a4e905f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -28,19 +28,19 @@ matrix:
 
     include:
     - env:
-        - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network)"
+        - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network and not clipboard)"
 
     - env:
-        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)"
+        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network and not clipboard)"
 
     - env:
-        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
+        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
       services:
         - mysql
         - postgresql
 
     - env:
-        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1"
+        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1"
       services:
         - mysql
         - postgresql
diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml
index 55e8e839f4fae..c9a2e4eefd19d 100644
--- a/ci/azure/posix.yml
+++ b/ci/azure/posix.yml
@@ -18,7 +18,7 @@ jobs:
         py36_minimum_versions:
           ENV_FILE: ci/deps/azure-36-minimum_versions.yaml
           CONDA_PY: "36"
-          PATTERN: "not slow and not network"
+          PATTERN: "not slow and not network and not clipboard"
 
         py36_locale_slow_old_np:
           ENV_FILE: ci/deps/azure-36-locale_slow.yaml
@@ -36,12 +36,12 @@ jobs:
           PATTERN: "not slow and not network"
           LANG: "it_IT.utf8"
           LC_ALL: "it_IT.utf8"
-          EXTRA_APT: "language-pack-it"
+          EXTRA_APT: "language-pack-it xsel"
 
         py36_32bit:
           ENV_FILE: ci/deps/azure-36-32bit.yaml
           CONDA_PY: "36"
-          PATTERN: "not slow and not network"
+          PATTERN: "not slow and not network and not clipboard"
           BITS32: "yes"
 
         py37_locale:
@@ -50,7 +50,7 @@ jobs:
           PATTERN: "not slow and not network"
           LANG: "zh_CN.utf8"
           LC_ALL: "zh_CN.utf8"
-          EXTRA_APT: "language-pack-zh-hans"
+          EXTRA_APT: "language-pack-zh-hans xsel"
 
         py37_np_dev:
           ENV_FILE: ci/deps/azure-37-numpydev.yaml
diff --git a/ci/run_tests.sh b/ci/run_tests.sh
index 8020680d617d7..0cb1f4aabf352 100755
--- a/ci/run_tests.sh
+++ b/ci/run_tests.sh
@@ -14,14 +14,14 @@ if [ "$COVERAGE" ]; then
     COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME"
 fi
 
-PYTEST_CMD="pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
-
-# Travis does not have have an X server
-if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-    DISPLAY=DISPLAY=:99.0
-    PYTEST_CMD="xvfb-run -e /dev/stdout $PYTEST_CMD"
+# If no X server is found, we use xvfb to emulate it
+if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then
+    export DISPLAY=":0"
+    XVFB="xvfb-run "
 fi
 
+PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
+
 echo $PYTEST_CMD
 sh -c "$PYTEST_CMD"
 
diff --git a/ci/setup_env.sh b/ci/setup_env.sh
index db28eaea8956e..e5bee09fe2f79 100755
--- a/ci/setup_env.sh
+++ b/ci/setup_env.sh
@@ -114,6 +114,11 @@ echo "remove postgres if has been installed with conda"
 echo "we use the one from the CI"
 conda remove postgresql -y --force || true
 
+echo
+echo "remove qt"
+echo "causes problems with the clipboard, we use xsel for that"
+conda remove qt -y --force || true
+
 echo
 echo "conda list pandas"
 conda list pandas
diff --git a/pandas/tests/io/test_clipboard.py b/pandas/tests/io/test_clipboard.py
index a69e5556f3e85..652cacaf14ffb 100644
--- a/pandas/tests/io/test_clipboard.py
+++ b/pandas/tests/io/test_clipboard.py
@@ -8,13 +8,7 @@
 from pandas import DataFrame, get_option, read_clipboard
 import pandas._testing as tm
 
-from pandas.io.clipboard import PyperclipException, clipboard_get, clipboard_set
-
-try:
-    DataFrame({"A": [1, 2]}).to_clipboard()
-    _DEPS_INSTALLED = 1
-except (PyperclipException, RuntimeError):
-    _DEPS_INSTALLED = 0
+from pandas.io.clipboard import clipboard_get, clipboard_set
 
 
 def build_kwargs(sep, excel):
@@ -148,7 +142,6 @@ def test_mock_clipboard(mock_clipboard):
 
 @pytest.mark.single
 @pytest.mark.clipboard
-@pytest.mark.skipif(not _DEPS_INSTALLED, reason="clipboard primitives not installed")
 @pytest.mark.usefixtures("mock_clipboard")
 class TestClipboard:
     def check_round_trip_frame(self, data, excel=None, sep=None, encoding=None):
@@ -256,9 +249,7 @@ def test_round_trip_valid_encodings(self, enc, df):
 
 @pytest.mark.single
 @pytest.mark.clipboard
-@pytest.mark.skipif(not _DEPS_INSTALLED, reason="clipboard primitives not installed")
 @pytest.mark.parametrize("data", ["\U0001f44d...", "Ωœ∑´...", "abcd..."])
-@pytest.mark.xfail(reason="flaky in CI", strict=False)
 def test_raw_roundtrip(data):
     # PR #25040 wide unicode wasn't copied correctly on PY3 on windows
     clipboard_set(data)