diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml
index 118a37e191673..97683e56915c6 100644
--- a/.github/workflows/posix.yml
+++ b/.github/workflows/posix.yml
@@ -26,6 +26,9 @@ jobs:
       matrix:
         env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml]
         pattern: ["not single_cpu", "single_cpu"]
+        # Don't test pyarrow v2/3: Causes timeouts in read_csv engine
+        # even if tests are skipped/xfailed
+        pyarrow_version: ["5", "6", "7"]
         include:
           - env_file: actions-38-downstream_compat.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -65,7 +68,7 @@ jobs:
       COVERAGE: ${{ !contains(matrix.env_file, 'pypy') }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.pyarrow_version || '' }}-${{ matrix.extra_apt || '' }}
       cancel-in-progress: true
 
     services:
@@ -133,6 +136,10 @@ jobs:
         use-only-tar-bz2: true
       if: ${{ env.IS_PYPY == 'false' }} # No pypy3.8 support
 
+    - name: Upgrade Arrow version
+      run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=${{ matrix.pyarrow_version }}
+      if: ${{ matrix.pyarrow_version }}
+
     - name: Setup PyPy
       uses: actions/setup-python@v2
       with:
diff --git a/pandas/tests/test_downstream.py b/pandas/tests/test_downstream.py
index 72ce6b837e810..ce7c7a634f5a9 100644
--- a/pandas/tests/test_downstream.py
+++ b/pandas/tests/test_downstream.py
@@ -176,7 +176,8 @@ def test_seaborn():
 
 
 def test_pandas_gbq():
-
+    # Older versions import from non-public, non-existent pandas funcs
+    pytest.importorskip("pandas_gbq", minversion="0.10.0")
     pandas_gbq = import_module("pandas_gbq")  # noqa:F841