diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml
index 0e8da7b66026f..35196ad2840c6 100644
--- a/.github/workflows/posix.yml
+++ b/.github/workflows/posix.yml
@@ -13,7 +13,6 @@ on:
       - "doc/**"
 
 env:
-  PYTEST_WORKERS: "auto"
   PANDAS_CI: 1
 
 jobs:
@@ -25,33 +24,48 @@ jobs:
     timeout-minutes: 120
     strategy:
       matrix:
-        settings: [
-          [actions-38-downstream_compat.yaml, "not slow and not network and not single_cpu", "", "", "", "", ""],
-          [actions-38-minimum_versions.yaml, "not single_cpu", "", "", "", "", ""],
-          [actions-38.yaml, "not slow and not network and not single_cpu", "language-pack-it", "it_IT.utf8", "it_IT.utf8", "", ""],
-          [actions-38.yaml, "not slow and not network and not single_cpu", "language-pack-zh-hans", "zh_CN.utf8", "zh_CN.utf8", "", ""],
-          [actions-38.yaml, "not single_cpu", "", "", "", "", ""],
-          [actions-pypy-38.yaml, "not slow and not single_cpu", "", "", "", "", "--max-worker-restart 0"],
-          [actions-39.yaml, "not single_cpu", "", "", "", "", ""],
-          [actions-310-numpydev.yaml, "not slow and not network and not single_cpu", "", "", "", "deprecate", "-W error"],
-          [actions-310.yaml, "not single_cpu", "", "", "", "", ""],
-        ]
+        env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml]
+        pattern: ["not single_cpu", "single_cpu"]
+        include:
+          - env_file: actions-38-downstream_compat.yaml
+            pattern: "not slow and not network and not single_cpu"
+            pytest_target: "pandas/tests/test_downstream.py"
+          - env_file: actions-38-minimum_versions.yaml
+            pattern: "not slow and not network and not single_cpu"
+          - env_file: actions-38.yaml
+            pattern: "not slow and not network and not single_cpu"
+            extra_apt: "language-pack-it"
+            lang: "it_IT.utf8"
+            lc_all: "it_IT.utf8"
+          - env_file: actions-38.yaml
+            pattern: "not slow and not network and not single_cpu"
+            extra_apt: "language-pack-zh-hans"
+            lang: "zh_CN.utf8"
+            lc_all: "zh_CN.utf8"
+          - env_file: actions-pypy-38.yaml
+            pattern: "not slow and not network and not single_cpu"
+            test_args: "--max-worker-restart 0"
+          - env_file: actions-310-numpydev.yaml
+            pattern: "not slow and not network and not single_cpu"
+            pandas_testing_mode: "deprecate"
+            test_args: "-W error"
       fail-fast: false
     env:
-      ENV_FILE: ci/deps/${{ matrix.settings[0] }}
-      PATTERN: ${{ matrix.settings[1] }}
-      EXTRA_APT: ${{ matrix.settings[2] }}
-      LANG: ${{ matrix.settings[3] }}
-      LC_ALL: ${{ matrix.settings[4] }}
-      PANDAS_TESTING_MODE: ${{ matrix.settings[5] }}
-      TEST_ARGS: ${{ matrix.settings[6] }}
-      PYTEST_TARGET:  pandas
-      IS_PYPY: ${{ contains(matrix.settings[0], 'pypy') }}
+      ENV_FILE: ci/deps/${{ matrix.env_file }}
+      PATTERN: ${{ matrix.pattern }}
+      EXTRA_APT: ${{ matrix.extra_apt || '' }}
+      LANG: ${{ matrix.lang || '' }}
+      LC_ALL: ${{ matrix.lc_all || '' }}
+      PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode || '' }}
+      TEST_ARGS: ${{ matrix.test_args || '' }}
+      PYTEST_WORKERS: ${{ contains(matrix.pattern, 'not single_cpu') && 'auto' || '1' }}
+      PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
+      IS_PYPY: ${{ contains(matrix.env_file, 'pypy') }}
       # TODO: re-enable coverage on pypy, its slow
-      COVERAGE: ${{ !contains(matrix.settings[0], 'pypy') }}
+      COVERAGE: ${{ !contains(matrix.env_file, 'pypy') }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.settings[0] }}-${{ matrix.settings[1] }}-${{ matrix.settings[2] }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}
       cancel-in-progress: true
 
     services:
diff --git a/pandas/tests/io/parser/conftest.py b/pandas/tests/io/parser/conftest.py
index 7428432aed6e5..066f448d97505 100644
--- a/pandas/tests/io/parser/conftest.py
+++ b/pandas/tests/io/parser/conftest.py
@@ -89,7 +89,7 @@ def csv1(datapath):
 
 _py_parsers_only = [_pythonParser]
 _c_parsers_only = [_cParserHighMemory, _cParserLowMemory]
-_pyarrow_parsers_only = [_pyarrowParser]
+_pyarrow_parsers_only = [pytest.param(_pyarrowParser, marks=pytest.mark.single_cpu)]
 
 _all_parsers = [*_c_parsers_only, *_py_parsers_only, *_pyarrow_parsers_only]
 
@@ -108,9 +108,8 @@ def all_parsers(request):
     parser = request.param()
     if parser.engine == "pyarrow":
         pytest.importorskip("pyarrow", VERSIONS["pyarrow"])
-        # Try setting num cpus to 1 to avoid hangs on Azure MacOS/Windows builds
-        # or better yet find a way to disable threads
-        # TODO(GH#44584) pytest.mark.single_cpu these tests
+        # Try finding a way to disable threads all together
+        # for more stable CI runs
         import pyarrow
 
         pyarrow.set_cpu_count(1)
@@ -149,8 +148,14 @@ def _get_all_parser_float_precision_combinations():
     params = []
     ids = []
     for parser, parser_id in zip(_all_parsers, _all_parser_ids):
+        if hasattr(parser, "values"):
+            # Wrapped in pytest.param, get the actual parser back
+            parser = parser.values[0]
         for precision in parser.float_precision_choices:
-            params.append((parser(), precision))
+            # Re-wrap in pytest.param for pyarrow
+            mark = pytest.mark.single_cpu if parser.engine == "pyarrow" else ()
+            param = pytest.param((parser(), precision), marks=mark)
+            params.append(param)
             ids.append(f"{parser_id}-{precision}")
 
     return {"params": params, "ids": ids}