diff --git a/pandas/conftest.py b/pandas/conftest.py
index 9009484f8d386..04f460902c11a 100644
--- a/pandas/conftest.py
+++ b/pandas/conftest.py
@@ -115,6 +115,12 @@ def pytest_collection_modifyitems(items, config):
     ]
 
     for item in items:
+        if config.getoption("--doctest-modules") or config.getoption(
+            "--doctest-cython", default=False
+        ):
+            # autouse=True for the add_doctest_imports can lead to expensive teardowns
+            # since doctest_namespace is a session fixture
+            item.add_marker(pytest.mark.usefixtures("add_doctest_imports"))
         # mark all tests in the pandas/tests/frame directory with "arraymanager"
         if "/frame/" in item.nodeid:
             item.add_marker(pytest.mark.arraymanager)
@@ -187,6 +193,15 @@ def pytest_collection_modifyitems(items, config):
     )
 
 
+@pytest.fixture
+def add_doctest_imports(doctest_namespace):
+    """
+    Make `np` and `pd` names available for doctests.
+    """
+    doctest_namespace["np"] = np
+    doctest_namespace["pd"] = pd
+
+
 # ----------------------------------------------------------------
 # Autouse fixtures
 # ----------------------------------------------------------------
@@ -198,15 +213,6 @@ def configure_tests():
     pd.set_option("chained_assignment", "raise")
 
 
-@pytest.fixture(autouse=True)
-def add_imports(doctest_namespace):
-    """
-    Make `np` and `pd` names available for doctests.
-    """
-    doctest_namespace["np"] = np
-    doctest_namespace["pd"] = pd
-
-
 # ----------------------------------------------------------------
 # Common arguments
 # ----------------------------------------------------------------