Fix Missing Fixture Teardown operations (#260)

When using the only_rerun and rerun_except queries (or both), the
plug-in was removing the teardown operations from the call-stack before
checking to see if the test should be re-run. This resulted in the
stack having all fixture operations removed that did not correspond
to a function fixture.

This commit adds a private variable to each test item that keeps
track of whether a test encountered a terminal error. The plugin now
checks if a test has encountered a terminal error before attempting
to clear the stack.

This commit fixes:
- #241
- #234

Author: HarryKrause

CHANGES.rst

@@ -4,6 +4,13 @@ Changelog
 14.0 (unreleased)
 -----------------
 
+Bug fixes
++++++++++
+
+- Fix missing teardown for non-function scoped fixtures when using only_rerun or rerun_except queries.
+  (`#234 <https://github.com/pytest-dev/pytest-rerunfailures/issues/234>`_)
+  and (`#241 <https://github.com/pytest-dev/pytest-rerunfailures/issues/241>`_)
+
 Breaking changes
 ++++++++++++++++
 

src/pytest_rerunfailures.py

@@ -481,7 +481,15 @@ def pytest_runtest_teardown(item, nextitem):
         return
 
     _test_failed_statuses = getattr(item, "_test_failed_statuses", {})
-    if item.execution_count <= reruns and any(_test_failed_statuses.values()):
+
+    # Only remove non-function level actions from the stack if the test is to be re-run
+    # Exceeding re-run limits, being free of failue statuses, and encountering
+    # allowable exceptions indicate that the test is not to be re-ran.
+    if (
+        item.execution_count <= reruns
+        and any(_test_failed_statuses.values())
+        and not any(item._terminal_errors.values())
+    ):
         # clean cashed results from any level of setups
         _remove_cached_results_from_failed_fixtures(item)
 
@@ -498,10 +506,16 @@ def pytest_runtest_makereport(item, call):
     if result.when == "setup":
         # clean failed statuses at the beginning of each test/rerun
         setattr(item, "_test_failed_statuses", {})
+
+        # create a dict to store error-check results for each stage
+        setattr(item, "_terminal_errors", {})
+
     _test_failed_statuses = getattr(item, "_test_failed_statuses", {})
     _test_failed_statuses[result.when] = result.failed
     item._test_failed_statuses = _test_failed_statuses
 
+    item._terminal_errors[result.when] = _should_hard_fail_on_error(item, result)
+
 
 def pytest_runtest_protocol(item, nextitem):
     """

tests/test_pytest_rerunfailures.py

@@ -1083,3 +1083,192 @@ def test_2():
 
     logging.info.assert_has_calls(expected_calls, any_order=False)
     assert_outcomes(result, failed=8, passed=2, rerun=18, skipped=5, error=1)
+
+
+def test_exception_matches_rerun_except_query(testdir):
+    testdir.makepyfile(
+        """
+        import pytest
+
+        @pytest.fixture(scope="session", autouse=True)
+        def session_fixture():
+            print("session setup")
+            yield "session"
+            print("session teardown")
+
+        @pytest.fixture(scope="package", autouse=True)
+        def package_fixture():
+            print("package setup")
+            yield "package"
+            print("package teardown")
+
+        @pytest.fixture(scope="module", autouse=True)
+        def module_fixture():
+            print("module setup")
+            yield "module"
+            print("module teardown")
+
+        @pytest.fixture(scope="class", autouse=True)
+        def class_fixture():
+            print("class setup")
+            yield "class"
+            print("class teardown")
+
+        @pytest.fixture(scope="function", autouse=True)
+        def function_fixture():
+            print("function setup")
+            yield "function"
+            print("function teardown")
+
+        @pytest.mark.flaky(reruns=1, rerun_except=["AssertionError"])
+        class TestStuff:
+            def test_1(self):
+                raise AssertionError("fail")
+
+            def test_2(self):
+                assert False
+
+    """
+    )
+    result = testdir.runpytest()
+    assert_outcomes(result, passed=0, failed=2, rerun=1)
+    result.stdout.fnmatch_lines("session teardown")
+    result.stdout.fnmatch_lines("package teardown")
+    result.stdout.fnmatch_lines("module teardown")
+    result.stdout.fnmatch_lines("class teardown")
+    result.stdout.fnmatch_lines("function teardown")
+
+
+def test_exception_not_match_rerun_except_query(testdir):
+    testdir.makepyfile(
+        """
+        import pytest
+
+        @pytest.fixture(scope="session", autouse=True)
+        def session_fixture():
+            print("session setup")
+            yield "session"
+            print("session teardown")
+
+        @pytest.fixture(scope="function", autouse=True)
+        def function_fixture():
+            print("function setup")
+            yield "function"
+            print("function teardown")
+
+        @pytest.mark.flaky(reruns=1, rerun_except="AssertionError")
+        def test_1(session_fixture, function_fixture):
+            raise ValueError("value")
+    """
+    )
+    result = testdir.runpytest()
+    assert_outcomes(result, passed=0, failed=1, rerun=1)
+    result.stdout.fnmatch_lines("session teardown")
+
+
+def test_exception_matches_only_rerun_query(testdir):
+    testdir.makepyfile(
+        """
+        import pytest
+
+        @pytest.fixture(scope="session", autouse=True)
+        def session_fixture():
+            print("session setup")
+            yield "session"
+            print("session teardown")
+
+        @pytest.fixture(scope="function", autouse=True)
+        def function_fixture():
+            print("function setup")
+            yield "function"
+            print("function teardown")
+
+        @pytest.mark.flaky(reruns=1, only_rerun=["AssertionError"])
+        def test_1(session_fixture, function_fixture):
+            raise AssertionError("fail")
+    """
+    )
+    result = testdir.runpytest()
+    assert_outcomes(result, passed=0, failed=1, rerun=1)
+    result.stdout.fnmatch_lines("session teardown")
+
+
+def test_exception_not_match_only_rerun_query(testdir):
+    testdir.makepyfile(
+        """
+        import pytest
+
+        @pytest.fixture(scope="session", autouse=True)
+        def session_fixture():
+            print("session setup")
+            yield "session"
+            print("session teardown")
+
+        @pytest.fixture(scope="function", autouse=True)
+        def function_fixture():
+            print("function setup")
+            yield "function"
+            print("function teardown")
+
+        @pytest.mark.flaky(reruns=1, only_rerun=["AssertionError"])
+        def test_1(session_fixture, function_fixture):
+            raise ValueError("fail")
+    """
+    )
+    result = testdir.runpytest()
+    assert_outcomes(result, passed=0, failed=1)
+    result.stdout.fnmatch_lines("session teardown")
+
+
+def test_exception_match_rerun_except_in_dual_query(testdir):
+    testdir.makepyfile(
+        """
+        import pytest
+
+        @pytest.fixture(scope="session", autouse=True)
+        def session_fixture():
+            print("session setup")
+            yield "session"
+            print("session teardown")
+
+        @pytest.fixture(scope="function", autouse=True)
+        def function_fixture():
+            print("function setup")
+            yield "function"
+            print("function teardown")
+
+        @pytest.mark.flaky(reruns=1, rerun_except=["Exception"], only_rerun=["Not"])
+        def test_1(session_fixture, function_fixture):
+            raise Exception("fail")
+    """
+    )
+    result = testdir.runpytest()
+    assert_outcomes(result, passed=0, failed=1)
+    result.stdout.fnmatch_lines("session teardown")
+
+
+def test_exception_match_only_rerun_in_dual_query(testdir):
+    testdir.makepyfile(
+        """
+        import pytest
+
+        @pytest.fixture(scope="session", autouse=True)
+        def session_fixture():
+            print("session setup")
+            yield "session"
+            print("session teardown")
+
+        @pytest.fixture(scope="function", autouse=True)
+        def function_fixture():
+            print("function setup")
+            yield "function"
+            print("function teardown")
+
+        @pytest.mark.flaky(reruns=1, rerun_except=["Not"], only_rerun=["Exception"])
+        def test_1(session_fixture, function_fixture):
+            raise Exception("fail")
+    """
+    )
+    result = testdir.runpytest()
+    assert_outcomes(result, passed=0, failed=1, rerun=1)
+    result.stdout.fnmatch_lines("session teardown")