Skip to content

Commit c648369

Browse files
authored
Use snapshot-tests to ease adapting assertions after changes to starter (#4310)
Prior to this commit, changing the tests in the jupiter-starter sample project required to manually adapt the expected XML string in `XmlAssertions` which was tedious and time-consuming. Instead, the XML report is now checked against a snapshot that is committed to version control. When a diff is encountered, the affected test will fail. The actual and expected snapshots can then be diffed and, if ok, the actual snapshot can be used to overwrite the expected one. To avoid committing host names and usernames to source control, they are obfuscated prior to writing the snapshot files.
1 parent b9e075c commit c648369

File tree

10 files changed

+512
-137
lines changed

10 files changed

+512
-137
lines changed

.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,7 @@ coverage.db*
3434
/.tool-versions
3535

3636
checksums*
37+
38+
# snapshot-tests
39+
*.snapshot_actual
40+
*.snapshot_raw

gradle/libs.versions.toml

+3
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ logback = "1.5.16"
1818
mockito = "5.15.2"
1919
opentest4j = "1.3.0"
2020
openTestReporting = "0.2.0-M2"
21+
snapshotTests = "1.11.0"
2122
surefire = "3.5.2"
2223
xmlunit = "2.10.0"
2324

@@ -64,6 +65,8 @@ openTestReporting-tooling-core = { module = "org.opentest4j.reporting:open-test-
6465
openTestReporting-tooling-spi = { module = "org.opentest4j.reporting:open-test-reporting-tooling-spi", version.ref = "openTestReporting" }
6566
picocli = { module = "info.picocli:picocli", version = "4.7.6" }
6667
slf4j-julBinding = { module = "org.slf4j:slf4j-jdk14", version = "2.0.16" }
68+
snapshotTests-junit5 = { module = "de.skuzzle.test:snapshot-tests-junit5", version.ref = "snapshotTests" }
69+
snapshotTests-xml = { module = "de.skuzzle.test:snapshot-tests-xml", version.ref = "snapshotTests" }
6770
spock1 = { module = "org.spockframework:spock-core", version = "1.3-groovy-2.5" }
6871
univocity-parsers = { module = "com.sonofab1rd:univocity-parsers", version = "2.10.1" }
6972
xmlunit-assertj = { module = "org.xmlunit:xmlunit-assertj3", version.ref = "xmlunit" }

platform-tooling-support-tests/platform-tooling-support-tests.gradle.kts

+2
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@ dependencies {
7474
testImplementation(libs.bundles.xmlunit)
7575
testImplementation(testFixtures(projects.junitJupiterApi))
7676
testImplementation(testFixtures(projects.junitPlatformReporting))
77+
testImplementation(libs.snapshotTests.junit5)
78+
testImplementation(libs.snapshotTests.xml)
7779

7880
thirdPartyJars(libs.junit4)
7981
thirdPartyJars(libs.assertj)

platform-tooling-support-tests/src/test/java/platform/tooling/support/tests/AntStarterTests.java

+8-2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@
1818
import java.nio.file.Path;
1919
import java.util.List;
2020

21+
import de.skuzzle.test.snapshots.Snapshot;
22+
import de.skuzzle.test.snapshots.junit5.EnableSnapshotTests;
23+
2124
import org.apache.tools.ant.Main;
2225
import org.junit.jupiter.api.Test;
2326
import org.junit.jupiter.api.Timeout;
@@ -29,11 +32,14 @@
2932
/**
3033
* @since 1.3
3134
*/
35+
@EnableSnapshotTests
3236
class AntStarterTests {
3337

3438
@Test
3539
@Timeout(60)
36-
void ant_starter(@TempDir Path workspace, @FilePrefix("ant") OutputFiles outputFiles) throws Exception {
40+
void ant_starter(@TempDir Path workspace, @FilePrefix("ant") OutputFiles outputFiles, Snapshot snapshot)
41+
throws Exception {
42+
3743
var result = ProcessStarters.java() //
3844
.workingDir(copyToWorkspace(Projects.JUPITER_STARTER, workspace)) //
3945
.addArguments("-cp", System.getProperty("antJars"), Main.class.getName()) //
@@ -57,6 +63,6 @@ void ant_starter(@TempDir Path workspace, @FilePrefix("ant") OutputFiles outputF
5763
result.stdOutLines());
5864

5965
var testResultsDir = workspace.resolve("build/test-report");
60-
verifyContainsExpectedStartedOpenTestReport(testResultsDir);
66+
verifyContainsExpectedStartedOpenTestReport(testResultsDir, snapshot);
6167
}
6268
}

platform-tooling-support-tests/src/test/java/platform/tooling/support/tests/GradleStarterTests.java

+8-2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@
1818

1919
import java.nio.file.Path;
2020

21+
import de.skuzzle.test.snapshots.Snapshot;
22+
import de.skuzzle.test.snapshots.junit5.EnableSnapshotTests;
23+
2124
import org.junit.jupiter.api.Test;
2225
import org.junit.jupiter.api.io.TempDir;
2326
import org.junit.platform.tests.process.OutputFiles;
@@ -30,10 +33,13 @@
3033
/**
3134
* @since 1.3
3235
*/
36+
@EnableSnapshotTests
3337
class GradleStarterTests {
3438

3539
@Test
36-
void gradle_wrapper(@TempDir Path workspace, @FilePrefix("gradle") OutputFiles outputFiles) throws Exception {
40+
void gradle_wrapper(@TempDir Path workspace, @FilePrefix("gradle") OutputFiles outputFiles, Snapshot snapshot)
41+
throws Exception {
42+
3743
var result = ProcessStarters.gradlew() //
3844
.workingDir(copyToWorkspace(Projects.JUPITER_STARTER, workspace)) //
3945
.addArguments("-Dmaven.repo=" + MavenRepo.dir()) //
@@ -47,6 +53,6 @@ void gradle_wrapper(@TempDir Path workspace, @FilePrefix("gradle") OutputFiles o
4753
assertThat(result.stdOut()).contains("Using Java version: 1.8");
4854

4955
var testResultsDir = workspace.resolve("build/test-results/test");
50-
verifyContainsExpectedStartedOpenTestReport(testResultsDir);
56+
verifyContainsExpectedStartedOpenTestReport(testResultsDir, snapshot);
5157
}
5258
}

platform-tooling-support-tests/src/test/java/platform/tooling/support/tests/MavenStarterTests.java

+8-3
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@
1818

1919
import java.nio.file.Path;
2020

21+
import de.skuzzle.test.snapshots.Snapshot;
22+
import de.skuzzle.test.snapshots.junit5.EnableSnapshotTests;
23+
2124
import org.junit.jupiter.api.Test;
2225
import org.junit.jupiter.api.io.TempDir;
2326
import org.junit.platform.tests.process.OutputFiles;
@@ -30,6 +33,7 @@
3033
/**
3134
* @since 1.3
3235
*/
36+
@EnableSnapshotTests
3337
class MavenStarterTests {
3438

3539
@ManagedResource
@@ -39,8 +43,9 @@ class MavenStarterTests {
3943
MavenRepoProxy mavenRepoProxy;
4044

4145
@Test
42-
void verifyJupiterStarterProject(@TempDir Path workspace, @FilePrefix("maven") OutputFiles outputFiles)
43-
throws Exception {
46+
void verifyJupiterStarterProject(@TempDir Path workspace, @FilePrefix("maven") OutputFiles outputFiles,
47+
Snapshot snapshot) throws Exception {
48+
4449
var result = ProcessStarters.maven(Helper.getJavaHome("8").orElseThrow(TestAbortedException::new)) //
4550
.workingDir(copyToWorkspace(Projects.JUPITER_STARTER, workspace)) //
4651
.addArguments(localMavenRepo.toCliArgument(), "-Dmaven.repo=" + MavenRepo.dir()) //
@@ -56,6 +61,6 @@ void verifyJupiterStarterProject(@TempDir Path workspace, @FilePrefix("maven") O
5661
assertThat(result.stdOut()).contains("Using Java version: 1.8");
5762

5863
var testResultsDir = workspace.resolve("target/surefire-reports");
59-
verifyContainsExpectedStartedOpenTestReport(testResultsDir);
64+
verifyContainsExpectedStartedOpenTestReport(testResultsDir, snapshot);
6065
}
6166
}

platform-tooling-support-tests/src/test/java/platform/tooling/support/tests/XmlAssertions.java

+56-130
Original file line numberDiff line numberDiff line change
@@ -10,147 +10,73 @@
1010

1111
package platform.tooling.support.tests;
1212

13+
import static de.skuzzle.test.snapshots.data.xml.XmlSnapshot.xml;
1314
import static org.junit.platform.reporting.testutil.FileUtils.findPath;
1415

16+
import java.io.IOException;
17+
import java.nio.file.Files;
1518
import java.nio.file.Path;
19+
import java.util.Map;
20+
import java.util.function.UnaryOperator;
1621
import java.util.regex.Pattern;
1722

18-
import org.xmlunit.assertj3.XmlAssert;
19-
import org.xmlunit.placeholder.PlaceholderDifferenceEvaluator;
23+
import de.skuzzle.test.snapshots.Snapshot;
24+
import de.skuzzle.test.snapshots.SnapshotSerializer;
25+
import de.skuzzle.test.snapshots.StructuredData;
26+
import de.skuzzle.test.snapshots.StructuredDataProvider;
2027

2128
class XmlAssertions {
2229

23-
static void verifyContainsExpectedStartedOpenTestReport(Path testResultsDir) {
30+
static void verifyContainsExpectedStartedOpenTestReport(Path testResultsDir, Snapshot snapshot) throws IOException {
2431
var xmlFile = findPath(testResultsDir, "glob:**/open-test-report.xml");
25-
verifyContent(xmlFile);
32+
verifyContent(xmlFile, snapshot);
2633
}
2734

28-
private static void verifyContent(Path xmlFile) {
29-
var expected = """
30-
<e:events xmlns="https://schemas.opentest4j.org/reporting/core/0.2.0"
31-
xmlns:e="https://schemas.opentest4j.org/reporting/events/0.2.0"
32-
xmlns:git="https://schemas.opentest4j.org/reporting/git/0.2.0"
33-
xmlns:java="https://schemas.opentest4j.org/reporting/java/0.2.0"
34-
xmlns:junit="https://schemas.junit.org/open-test-reporting"
35-
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
36-
xsi:schemaLocation="https://schemas.junit.org/open-test-reporting https://junit.org/junit5/schemas/open-test-reporting/junit-1.9.xsd">
37-
<infrastructure>
38-
<hostName>${xmlunit.ignore}</hostName>
39-
<userName>${xmlunit.ignore}</userName>
40-
<operatingSystem>${xmlunit.ignore}</operatingSystem>
41-
<cpuCores>${xmlunit.ignore}</cpuCores>
42-
<java:javaVersion>${xmlunit.ignore}</java:javaVersion>
43-
<java:fileEncoding>${xmlunit.ignore}</java:fileEncoding>
44-
<java:heapSize max="${xmlunit.isNumber}"/>
45-
</infrastructure>
46-
<e:started id="1" name="JUnit Jupiter" time="${xmlunit.isDateTime}">
47-
<metadata>
48-
<junit:uniqueId>[engine:junit-jupiter]</junit:uniqueId>
49-
<junit:legacyReportingName>JUnit Jupiter</junit:legacyReportingName>
50-
<junit:type>CONTAINER</junit:type>
51-
</metadata>
52-
</e:started>
53-
<e:started id="2" name="CalculatorTests" parentId="1" time="${xmlunit.isDateTime}">
54-
<metadata>
55-
<junit:uniqueId>[engine:junit-jupiter]/[class:com.example.project.CalculatorTests]</junit:uniqueId>
56-
<junit:legacyReportingName>com.example.project.CalculatorTests</junit:legacyReportingName>
57-
<junit:type>CONTAINER</junit:type>
58-
</metadata>
59-
<sources>
60-
<java:classSource className="com.example.project.CalculatorTests"/>
61-
</sources>
62-
</e:started>
63-
<e:started id="3" name="1 + 1 = 2" parentId="2" time="${xmlunit.isDateTime}">
64-
<metadata>
65-
<junit:uniqueId>[engine:junit-jupiter]/[class:com.example.project.CalculatorTests]/[method:addsTwoNumbers()]</junit:uniqueId>
66-
<junit:legacyReportingName>addsTwoNumbers()</junit:legacyReportingName>
67-
<junit:type>TEST</junit:type>
68-
</metadata>
69-
<sources>
70-
<java:methodSource className="com.example.project.CalculatorTests" methodName="addsTwoNumbers" methodParameterTypes=""/>
71-
</sources>
72-
</e:started>
73-
<e:finished id="3" time="${xmlunit.isDateTime}">
74-
<result status="SUCCESSFUL"/>
75-
</e:finished>
76-
<e:started id="4" name="add(int, int, int)" parentId="2" time="${xmlunit.isDateTime}">
77-
<metadata>
78-
<junit:uniqueId>[engine:junit-jupiter]/[class:com.example.project.CalculatorTests]/[test-template:add(int, int, int)]</junit:uniqueId>
79-
<junit:legacyReportingName>add(int, int, int)</junit:legacyReportingName>
80-
<junit:type>CONTAINER</junit:type>
81-
</metadata>
82-
<sources>
83-
<java:methodSource className="com.example.project.CalculatorTests" methodName="add" methodParameterTypes="int, int, int"/>
84-
</sources>
85-
</e:started>
86-
<e:started id="5" name="0 + 1 = 1" parentId="4" time="${xmlunit.isDateTime}">
87-
<metadata>
88-
<junit:uniqueId>[engine:junit-jupiter]/[class:com.example.project.CalculatorTests]/[test-template:add(int, int, int)]/[test-template-invocation:#1]</junit:uniqueId>
89-
<junit:legacyReportingName>add(int, int, int)[1]</junit:legacyReportingName>
90-
<junit:type>TEST</junit:type>
91-
</metadata>
92-
<sources>
93-
<java:methodSource className="com.example.project.CalculatorTests" methodName="add" methodParameterTypes="int, int, int"/>
94-
</sources>
95-
</e:started>
96-
<e:finished id="5" time="${xmlunit.isDateTime}">
97-
<result status="SUCCESSFUL"/>
98-
</e:finished>
99-
<e:started id="6" name="1 + 2 = 3" parentId="4" time="${xmlunit.isDateTime}">
100-
<metadata>
101-
<junit:uniqueId>[engine:junit-jupiter]/[class:com.example.project.CalculatorTests]/[test-template:add(int, int, int)]/[test-template-invocation:#2]</junit:uniqueId>
102-
<junit:legacyReportingName>add(int, int, int)[2]</junit:legacyReportingName>
103-
<junit:type>TEST</junit:type>
104-
</metadata>
105-
<sources>
106-
<java:methodSource className="com.example.project.CalculatorTests" methodName="add" methodParameterTypes="int, int, int"/>
107-
</sources>
108-
</e:started>
109-
<e:finished id="6" time="${xmlunit.isDateTime}">
110-
<result status="SUCCESSFUL"/>
111-
</e:finished>
112-
<e:started id="7" name="49 + 51 = 100" parentId="4" time="${xmlunit.isDateTime}">
113-
<metadata>
114-
<junit:uniqueId>[engine:junit-jupiter]/[class:com.example.project.CalculatorTests]/[test-template:add(int, int, int)]/[test-template-invocation:#3]</junit:uniqueId>
115-
<junit:legacyReportingName>add(int, int, int)[3]</junit:legacyReportingName>
116-
<junit:type>TEST</junit:type>
117-
</metadata>
118-
<sources>
119-
<java:methodSource className="com.example.project.CalculatorTests" methodName="add" methodParameterTypes="int, int, int"/>
120-
</sources>
121-
</e:started>
122-
<e:finished id="7" time="${xmlunit.isDateTime}">
123-
<result status="SUCCESSFUL"/>
124-
</e:finished>
125-
<e:started id="8" name="1 + 100 = 101" parentId="4" time="${xmlunit.isDateTime}">
126-
<metadata>
127-
<junit:uniqueId>[engine:junit-jupiter]/[class:com.example.project.CalculatorTests]/[test-template:add(int, int, int)]/[test-template-invocation:#4]</junit:uniqueId>
128-
<junit:legacyReportingName>add(int, int, int)[4]</junit:legacyReportingName>
129-
<junit:type>TEST</junit:type>
130-
</metadata>
131-
<sources>
132-
<java:methodSource className="com.example.project.CalculatorTests" methodName="add" methodParameterTypes="int, int, int"/>
133-
</sources>
134-
</e:started>
135-
<e:finished id="8" time="${xmlunit.isDateTime}">
136-
<result status="SUCCESSFUL"/>
137-
</e:finished>
138-
<e:finished id="4" time="${xmlunit.isDateTime}">
139-
<result status="SUCCESSFUL"/>
140-
</e:finished>
141-
<e:finished id="2" time="${xmlunit.isDateTime}">
142-
<result status="SUCCESSFUL"/>
143-
</e:finished>
144-
<e:finished id="1" time="${xmlunit.isDateTime}">
145-
<result status="SUCCESSFUL"/>
146-
</e:finished>
147-
</e:events>
148-
""";
35+
private static void verifyContent(Path xmlFile, Snapshot snapshot) throws IOException {
36+
snapshot.named("open-test-report.xml") //
37+
.assertThat(Files.readString(xmlFile)) //
38+
.as(obfuscated( //
39+
xml() //
40+
.withXPathNamespaceContext(Map.of( //
41+
"c", "https://schemas.opentest4j.org/reporting/core/0.2.0", //
42+
"e", "https://schemas.opentest4j.org/reporting/events/0.2.0", //
43+
"java", "https://schemas.opentest4j.org/reporting/java/0.2.0" //
44+
)) //
45+
.withComparisonRules(rules -> rules //
46+
.pathAt("//@time").mustMatch(
47+
Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?Z")) //
48+
.pathAt("//c:infrastructure/c:hostName/text()").ignore() //
49+
.pathAt("//c:infrastructure/c:userName/text()").ignore() //
50+
.pathAt("//c:infrastructure/c:operatingSystem/text()").ignore() //
51+
.pathAt("//c:infrastructure/c:cpuCores/text()").ignore() //
52+
.pathAt("//c:infrastructure/java:javaVersion/text()").ignore() //
53+
.pathAt("//c:infrastructure/java:fileEncoding/text()").ignore() //
54+
.pathAt("//c:infrastructure/java:heapSize/@max").ignore() //
55+
), //
56+
text -> text //
57+
.replaceAll("<hostName>.+?</hostName>", "<hostName>obfuscated</hostName>") //
58+
.replaceAll("<userName>.+?</userName>", "<userName>obfuscated</userName>") //
59+
)) //
60+
.matchesSnapshotStructure();
61+
}
62+
63+
private static StructuredDataProvider obfuscated(StructuredDataProvider provider,
64+
UnaryOperator<String> obfuscator) {
65+
return () -> {
66+
var structuredData = provider.build();
67+
var snapshotSerializer = obfuscatingSnapshotSerializer(structuredData.snapshotSerializer(), obfuscator);
68+
return StructuredData.with(snapshotSerializer, structuredData.structuralAssertions());
69+
};
70+
}
14971

150-
XmlAssert.assertThat(xmlFile).and(expected) //
151-
.withDifferenceEvaluator(new PlaceholderDifferenceEvaluator(Pattern.quote("${"), Pattern.quote("}"),
152-
Pattern.quote("#"), Pattern.quote("#"), ",")) //
153-
.ignoreWhitespace() //
154-
.areIdentical();
72+
private static SnapshotSerializer obfuscatingSnapshotSerializer(SnapshotSerializer delegate,
73+
UnaryOperator<String> obfuscator) {
74+
return testResult -> {
75+
Object obfuscatedTestResult = testResult;
76+
if (testResult instanceof String) {
77+
obfuscatedTestResult = obfuscator.apply((String) testResult);
78+
}
79+
return delegate.serialize(obfuscatedTestResult);
80+
};
15581
}
15682
}

0 commit comments

Comments
 (0)