Move context failure tracking to the ContextCache

In the previous commit which introduced the new context failure threshold
support in the TestContext framework, the context failure tracking was
tied to an instance of DefaultCacheAwareContextLoaderDelegate.
Consequently, the feature was only supported within a given test class.

This commit therefore moves context failure tracking to the ContextCache
SPI (and DefaultContextCache) so that the feature applies to all test
classes within the current test suite (i.e., JVM).

This commit also includes the total failure count in the statistics
logged by the DefaultContextCache.

See gh-14182

Author: sbrannen

spring-test/src/main/java/org/springframework/test/context/CacheAwareContextLoaderDelegate.java

@@ -103,7 +103,9 @@ default boolean isContextLoaded(MergedContextConfiguration mergedConfig) {
 	 * context consistently fails to load — for example, due to a configuration
 	 * error that prevents the context from successfully loading — this
 	 * method should preemptively throw an {@link IllegalStateException} if the
-	 * configured failure threshold has been exceeded.
+	 * configured failure threshold has been exceeded. Note that the {@code ContextCache}
+	 * provides support for tracking and incrementing the failure count for a given
+	 * context cache key.
 	 * <p>The cache statistics should be logged by invoking
 	 * {@link org.springframework.test.context.cache.ContextCache#logStatistics()}.
 	 * @param mergedConfig the merged context configuration to use to load the

spring-test/src/main/java/org/springframework/test/context/cache/ContextCache.java

@@ -31,6 +31,11 @@
  * with a {@linkplain ContextCacheUtils#retrieveMaxCacheSize maximum size} and
  * a custom eviction policy.
  *
+ * <p>As of Spring Framework 6.1, this SPI includes optional support for
+ * {@linkplain #getFailureCount(MergedContextConfiguration) tracking} and
+ * {@linkplain #incrementFailureCount(MergedContextConfiguration) incrementing}
+ * failure counts.
+ *
  * <h3>Rationale</h3>
  * <p>Context caching can have significant performance benefits if context
  * initialization is complex. Although the initialization of a Spring context
@@ -117,6 +122,37 @@ public interface ContextCache {
 	 */
 	void remove(MergedContextConfiguration key, @Nullable HierarchyMode hierarchyMode);
 
+	/**
+	 * Get the failure count for the given key.
+	 * <p>A <em>failure</em> is any attempt to load the {@link ApplicationContext}
+	 * for the given key that results in an exception.
+	 * <p>The default implementation of this method always returns {@code 0}.
+	 * Concrete implementations are therefore highly encouraged to override this
+	 * method and {@link #incrementFailureCount(MergedContextConfiguration)} with
+	 * appropriate behavior. Note that the standard {@code ContextContext}
+	 * implementation in Spring overrides these methods appropriately.
+	 * @param key the context key; never {@code null}
+	 * @since 6.1
+	 * @see #incrementFailureCount(MergedContextConfiguration)
+	 */
+	default int getFailureCount(MergedContextConfiguration key) {
+		return 0;
+	}
+
+	/**
+	 * Increment the failure count for the given key.
+	 * <p>The default implementation of this method does nothing. Concrete
+	 * implementations are therefore highly encouraged to override this
+	 * method and {@link #getFailureCount(MergedContextConfiguration)} with
+	 * appropriate behavior. Note that the standard {@code ContextContext}
+	 * implementation in Spring overrides these methods appropriately.
+	 * @param key the context key; never {@code null}
+	 * @since 6.1
+	 * @see #getFailureCount(MergedContextConfiguration)
+	 */
+	default void incrementFailureCount(MergedContextConfiguration key) {
+	}
+
 	/**
 	 * Determine the number of contexts currently stored in the cache.
 	 * <p>If the cache contains more than {@code Integer.MAX_VALUE} elements,

spring-test/src/main/java/org/springframework/test/context/cache/DefaultCacheAwareContextLoaderDelegate.java

@@ -16,9 +16,7 @@
 
 package org.springframework.test.context.cache;
 
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -80,12 +78,6 @@ public class DefaultCacheAwareContextLoaderDelegate implements CacheAwareContext
 
 	private final ContextCache contextCache;
 
-	/**
-	 * Map of context keys to context load failure counts.
-	 * @since 6.1
-	 */
-	private final Map<MergedContextConfiguration, Integer> failureCounts = new HashMap<>(32);
-
 	/**
 	 * The configured failure threshold for errors encountered while attempting to
 	 * load an {@link ApplicationContext}.
@@ -144,7 +136,7 @@ public ApplicationContext loadContext(MergedContextConfiguration mergedConfig) {
 		synchronized (this.contextCache) {
 			ApplicationContext context = this.contextCache.get(mergedConfig);
 			if (context == null) {
-				Integer failureCount = this.failureCounts.getOrDefault(mergedConfig, 0);
+				Integer failureCount = this.contextCache.getFailureCount(mergedConfig);
 				if (failureCount >= this.failureThreshold) {
 					throw new IllegalStateException("""
 							ApplicationContext failure threshold (%d) exceeded: \
@@ -165,7 +157,7 @@ ApplicationContext failure threshold (%d) exceeded: \
 					this.contextCache.put(mergedConfig, context);
 				}
 				catch (Exception ex) {
-					this.failureCounts.put(mergedConfig, ++failureCount);
+					this.contextCache.incrementFailureCount(mergedConfig);
 					Throwable cause = ex;
 					if (ex instanceof ContextLoadException cle) {
 						cause = cle.getCause();

spring-test/src/main/java/org/springframework/test/context/cache/DefaultContextCache.java

@@ -72,6 +72,14 @@ public class DefaultContextCache implements ContextCache {
 	private final Map<MergedContextConfiguration, Set<MergedContextConfiguration>> hierarchyMap =
 			new ConcurrentHashMap<>(32);
 
+	/**
+	 * Map of context keys to context load failure counts.
+	 * @since 6.1
+	 */
+	private final Map<MergedContextConfiguration, Integer> failureCounts = new ConcurrentHashMap<>(32);
+
+	private final AtomicInteger totalFailureCount = new AtomicInteger();
+
 	private final int maxSize;
 
 	private final AtomicInteger hitCount = new AtomicInteger();
@@ -209,6 +217,23 @@ private void remove(List<MergedContextConfiguration> removedContexts, MergedCont
 		removedContexts.add(key);
 	}
 
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public int getFailureCount(MergedContextConfiguration key) {
+		return this.failureCounts.getOrDefault(key, 0);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public void incrementFailureCount(MergedContextConfiguration key) {
+		this.totalFailureCount.incrementAndGet();
+		this.failureCounts.merge(key, 1, Integer::sum);
+	}
+
 	/**
 	 * {@inheritDoc}
 	 */
@@ -256,6 +281,8 @@ public void reset() {
 		synchronized (this.contextMap) {
 			clear();
 			clearStatistics();
+			this.totalFailureCount.set(0);
+			this.failureCounts.clear();
 		}
 	}
 
@@ -306,6 +333,7 @@ public String toString() {
 				.append("parentContextCount", getParentContextCount())
 				.append("hitCount", getHitCount())
 				.append("missCount", getMissCount())
+				.append("failureCount", this.totalFailureCount)
 				.toString();
 	}
 

spring-test/src/test/java/org/springframework/test/context/cache/ContextFailureThresholdTests.java

@@ -18,13 +18,9 @@
 
 import java.util.concurrent.atomic.AtomicInteger;
 
-import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.MethodOrderer.OrderAnnotation;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestMethodOrder;
 import org.junit.platform.testkit.engine.EngineTestKit;
 
 import org.springframework.context.annotation.Bean;
@@ -49,125 +45,102 @@
  */
 class ContextFailureThresholdTests {
 
-	private static final AtomicInteger loadCount = new AtomicInteger(0);
+	private static final AtomicInteger passingLoadCount = new AtomicInteger(0);
+	private static final AtomicInteger failingLoadCount = new AtomicInteger(0);
 
 
 	@BeforeEach
 	@AfterEach
 	void resetFlag() {
-		loadCount.set(0);
+		resetContextCache();
+		passingLoadCount.set(0);
+		failingLoadCount.set(0);
 		SpringProperties.setProperty(CONTEXT_FAILURE_THRESHOLD_PROPERTY_NAME, null);
 	}
 
 	@Test
 	void defaultThreshold() {
-		assertThat(loadCount.get()).isZero();
-
-		EngineTestKit.engine("junit-jupiter")//
-				.selectors(selectClass(PassingTestCase.class))// 2 passing
-				.selectors(selectClass(FailingTestCase.class))// 3 failing
-				.execute()//
-				.testEvents()//
-				.assertStatistics(stats -> stats.started(5).succeeded(2).failed(3));
-		assertThat(loadCount.get()).isEqualTo(DEFAULT_CONTEXT_FAILURE_THRESHOLD);
+		runTests();
+		assertThat(passingLoadCount.get()).isEqualTo(1);
+		assertThat(failingLoadCount.get()).isEqualTo(DEFAULT_CONTEXT_FAILURE_THRESHOLD);
 	}
 
 	@Test
 	void customThreshold() {
-		assertThat(loadCount.get()).isZero();
+		int customThreshold = 2;
+		SpringProperties.setProperty(CONTEXT_FAILURE_THRESHOLD_PROPERTY_NAME, Integer.toString(customThreshold));
 
-		int threshold = 2;
-		SpringProperties.setProperty(CONTEXT_FAILURE_THRESHOLD_PROPERTY_NAME, Integer.toString(threshold));
-
-		EngineTestKit.engine("junit-jupiter")//
-				.selectors(selectClass(PassingTestCase.class))// 2 passing
-				.selectors(selectClass(FailingTestCase.class))// 3 failing
-				.execute()//
-				.testEvents()//
-				.assertStatistics(stats -> stats.started(5).succeeded(2).failed(3));
-		assertThat(loadCount.get()).isEqualTo(threshold);
+		runTests();
+		assertThat(passingLoadCount.get()).isEqualTo(1);
+		assertThat(failingLoadCount.get()).isEqualTo(customThreshold);
 	}
 
 	@Test
 	void thresholdEffectivelyDisabled() {
-		assertThat(loadCount.get()).isZero();
-
 		SpringProperties.setProperty(CONTEXT_FAILURE_THRESHOLD_PROPERTY_NAME, "999999");
 
+		runTests();
+		assertThat(passingLoadCount.get()).isEqualTo(1);
+		assertThat(failingLoadCount.get()).isEqualTo(6);
+	}
+
+	private static void runTests() {
 		EngineTestKit.engine("junit-jupiter")//
-				.selectors(selectClass(PassingTestCase.class))// 2 passing
-				.selectors(selectClass(FailingTestCase.class))// 3 failing
-				.execute()//
-				.testEvents()//
-				.assertStatistics(stats -> stats.started(5).succeeded(2).failed(3));
-		assertThat(loadCount.get()).isEqualTo(3);
+			.selectors(//
+				selectClass(PassingTestCase.class), // 3 passing
+				selectClass(FailingConfigTestCase.class), // 3 failing
+				selectClass(SharedFailingConfigTestCase.class) // 3 failing
+			)//
+			.execute()//
+			.testEvents()//
+			.assertStatistics(stats -> stats.started(9).succeeded(3).failed(6));
+		assertContextCacheStatistics(1, 2, (1 + 3 + 3));
 	}
 
 
-	@SpringJUnitConfig
 	@TestExecutionListeners(DependencyInjectionTestExecutionListener.class)
-	static class PassingTestCase {
-
-		@BeforeAll
-		static void verifyInitialCacheState() {
-			resetContextCache();
-			assertContextCacheStatistics("BeforeAll", 0, 0, 0);
-		}
-
-		@AfterAll
-		static void verifyFinalCacheState() {
-			assertContextCacheStatistics("AfterAll", 1, 1, 1);
-			resetContextCache();
-		}
+	static abstract class BaseTestCase {
 
 		@Test
 		void test1() {}
 
 		@Test
 		void test2() {}
 
-		@Configuration
-		static class PassingConfig {
-		}
+		@Test
+		void test3() {}
 	}
 
-	@SpringJUnitConfig
-	@TestExecutionListeners(DependencyInjectionTestExecutionListener.class)
-	@TestMethodOrder(OrderAnnotation.class)
-	static class FailingTestCase {
-
-		@BeforeAll
-		static void verifyInitialCacheState() {
-			resetContextCache();
-			assertContextCacheStatistics("BeforeAll", 0, 0, 0);
-		}
+	@SpringJUnitConfig(PassingConfig.class)
+	static class PassingTestCase extends BaseTestCase {
+	}
 
-		@AfterAll
-		static void verifyFinalCacheState() {
-			assertContextCacheStatistics("AfterAll", 0, 0, 3);
-			resetContextCache();
-		}
+	@SpringJUnitConfig(FailingConfig.class)
+	static class FailingConfigTestCase extends BaseTestCase {
+	}
 
-		@Test
-		void test1() {}
+	@SpringJUnitConfig(FailingConfig.class)
+	static class SharedFailingConfigTestCase extends BaseTestCase {
+	}
 
-		@Test
-		void test2() {}
+	@Configuration
+	static class PassingConfig {
 
-		@Test
-		void test3() {}
+		PassingConfig() {
+			passingLoadCount.incrementAndGet();
+		}
+	}
 
-		@Configuration
-		static class FailingConfig {
+	@Configuration
+	static class FailingConfig {
 
-			FailingConfig() {
-				loadCount.incrementAndGet();
-			}
+		FailingConfig() {
+			failingLoadCount.incrementAndGet();
+		}
 
-			@Bean
-			String explosiveString() {
-				throw new RuntimeException("Boom!");
-			}
+		@Bean
+		String explosiveString() {
+			throw new RuntimeException("Boom!");
 		}
 	}