Skip to content

Commit 11d0541

Browse files
andpabchalmagr
andpab
and
chalmagr
authored
[SUREFIRE-2065] Fix parameterized JUnit4 test reporting (#608)
* [SUREFIRE-2065] Fix parameterized JUnit4 test reporting --------- Co-authored-by: Christian Marquez Grabia <[email protected]>
1 parent b8887b4 commit 11d0541

File tree

14 files changed

+579
-14
lines changed

14 files changed

+579
-14
lines changed

‎surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/TestFile.java

+19
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,25 @@ public TestFile assertContainsText( String text )
155155
return assertContainsText( containsString( text ) );
156156
}
157157

158+
public TestFile assertNotContainsText( Matcher<String> matcher )
159+
{
160+
final List<String> list = surefireVerifier.loadFile( file, encoding );
161+
for ( String line : list )
162+
{
163+
if ( matcher.matches( line ) )
164+
{
165+
Assert.fail( "Found unexpected message in log" );
166+
return null;
167+
}
168+
}
169+
return this;
170+
}
171+
172+
public TestFile assertNotContainsText( String text )
173+
{
174+
return assertNotContainsText( containsString( text ) );
175+
}
176+
158177
public URI toURI()
159178
{
160179
return file.toURI();

‎surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2065IT.java

+103
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package org.apache.maven.surefire.its.jiras;
2+
3+
/*
4+
* Licensed to the Apache Software Foundation (ASF) under one
5+
* or more contributor license agreements. See the NOTICE file
6+
* distributed with this work for additional information
7+
* regarding copyright ownership. The ASF licenses this file
8+
* to you under the Apache License, Version 2.0 (the
9+
* "License"); you may not use this file except in compliance
10+
* with the License. You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing,
15+
* software distributed under the License is distributed on an
16+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
* KIND, either express or implied. See the License for the
18+
* specific language governing permissions and limitations
19+
* under the License.
20+
*/
21+
22+
import static java.nio.charset.StandardCharsets.UTF_8;
23+
24+
import org.apache.maven.it.VerificationException;
25+
import org.apache.maven.surefire.its.fixture.OutputValidator;
26+
import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
27+
import org.hamcrest.Matchers;
28+
import org.junit.Test;
29+
30+
/**
31+
* Integration Tests for SUREFIRE-2065
32+
*/
33+
@SuppressWarnings( "checkstyle:magicnumber" )
34+
public class Surefire2065IT extends SurefireJUnit4IntegrationTestCase
35+
{
36+
@Test
37+
public void shouldNotDetectFlakyTestsWhenCombiningJunit4And5Tests() throws VerificationException
38+
{
39+
OutputValidator validator = unpack( "surefire-2065-common" )
40+
.mavenTestFailureIgnore( true )
41+
.executeTest()
42+
.assertTestSuiteResults( 8, 0, 4, 0, 0 );
43+
44+
assertJunit4( validator );
45+
assertJunit5( validator );
46+
}
47+
48+
@Test
49+
public void shouldNotDetectFlakyTestsWhenRunningOnlyJunit4() throws VerificationException
50+
{
51+
OutputValidator validator = unpack( "surefire-2065-junit4" )
52+
.mavenTestFailureIgnore( true )
53+
.executeTest()
54+
.assertTestSuiteResults( 4, 0, 2, 0, 0 );
55+
56+
assertJunit4( validator );
57+
}
58+
59+
@Test
60+
public void shouldNotDetectFlakyTestsWhenRunningOnlyJunit5() throws VerificationException
61+
{
62+
OutputValidator validator = unpack( "surefire-2065-junit5" )
63+
.mavenTestFailureIgnore( true )
64+
.executeTest()
65+
.assertTestSuiteResults( 4, 0, 2, 0, 0 );
66+
67+
assertJunit5( validator );
68+
}
69+
70+
private static void assertJunit4( OutputValidator validator )
71+
{
72+
validator.getSurefireReportsFile( "TEST-pkg.junit4.ParameterizedTest.xml", UTF_8 )
73+
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\[0]\".*/>$" ) )
74+
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\[1]\".*[^/]>$" ) )
75+
.assertContainsText( "<failure message=" )
76+
.assertContainsText( "<rerunFailure message=" )
77+
.assertNotContainsText( "<flakyFailure message=" );
78+
79+
validator.getSurefireReportsFile( "TEST-pkg.junit4.ParameterizedWithDisplayNameTest.xml", UTF_8 )
80+
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\[value=0]\".*/>$" ) )
81+
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\[value=1]\".*[^/]>$" ) )
82+
.assertContainsText( "<failure message=" )
83+
.assertContainsText( "<rerunFailure message=" )
84+
.assertNotContainsText( "<flakyFailure message=" );
85+
}
86+
87+
private static void assertJunit5( OutputValidator validator )
88+
{
89+
validator.getSurefireReportsFile( "TEST-pkg.junit5.ParameterizedTest.xml", UTF_8 )
90+
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\(int\\)\\[1]\".*/>$" ) )
91+
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\(int\\)\\[2]\".*[^/]>$" ) )
92+
.assertContainsText( "<failure message=" )
93+
.assertContainsText( "<rerunFailure message=" )
94+
.assertNotContainsText( "<flakyFailure message=" );
95+
96+
validator.getSurefireReportsFile( "TEST-pkg.junit5.ParameterizedWithDisplayNameTest.xml", UTF_8 )
97+
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\(int\\)\\[1]\".*/>$" ) )
98+
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\(int\\)\\[2]\".*[^/]>$" ) )
99+
.assertContainsText( "<failure message=" )
100+
.assertContainsText( "<rerunFailure message=" )
101+
.assertNotContainsText( "<flakyFailure message=" );
102+
}
103+
}

‎surefire-its/src/test/resources/surefire-2065-common/pom.xml

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one
4+
~ or more contributor license agreements. See the NOTICE file
5+
~ distributed with this work for additional information
6+
~ regarding copyright ownership. The ASF licenses this file
7+
~ to you under the Apache License, Version 2.0 (the
8+
~ "License"); you may not use this file except in compliance
9+
~ with the License. You may obtain a copy of the License at
10+
~
11+
~ http://www.apache.org/licenses/LICENSE-2.0
12+
~
13+
~ Unless required by applicable law or agreed to in writing,
14+
~ software distributed under the License is distributed on an
15+
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
~ KIND, either express or implied. See the License for the
17+
~ specific language governing permissions and limitations
18+
~ under the License.
19+
-->
20+
21+
<project xmlns="http://maven.apache.org/POM/4.0.0"
22+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
23+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
24+
<modelVersion>4.0.0</modelVersion>
25+
26+
<groupId>org.example</groupId>
27+
<artifactId>surefire-2065-common</artifactId>
28+
<version>1.0-SNAPSHOT</version>
29+
30+
<properties>
31+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
32+
<maven.compiler.source>${java.specification.version}</maven.compiler.source>
33+
<maven.compiler.target>${java.specification.version}</maven.compiler.target>
34+
<junit4.version>4.12</junit4.version>
35+
<junit5.version>5.3.2</junit5.version>
36+
</properties>
37+
38+
<dependencies>
39+
<dependency>
40+
<groupId>junit</groupId>
41+
<artifactId>junit</artifactId>
42+
<version>${junit4.version}</version>
43+
<scope>test</scope>
44+
</dependency>
45+
<dependency>
46+
<groupId>org.junit.jupiter</groupId>
47+
<artifactId>junit-jupiter-api</artifactId>
48+
<version>${junit5.version}</version>
49+
<scope>test</scope>
50+
</dependency>
51+
<dependency>
52+
<groupId>org.junit.jupiter</groupId>
53+
<artifactId>junit-jupiter-params</artifactId>
54+
<version>${junit5.version}</version>
55+
<scope>test</scope>
56+
</dependency>
57+
</dependencies>
58+
59+
<build>
60+
<plugins>
61+
<plugin>
62+
<groupId>org.apache.maven.plugins</groupId>
63+
<artifactId>maven-surefire-plugin</artifactId>
64+
<version>${surefire.version}</version>
65+
<configuration>
66+
<rerunFailingTestsCount>1</rerunFailingTestsCount>
67+
</configuration>
68+
</plugin>
69+
</plugins>
70+
</build>
71+
72+
</project>

‎surefire-its/src/test/resources/surefire-2065-common/src/test/java/pkg/junit4/ParameterizedTest.java

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package pkg.junit4;
2+
3+
import static org.junit.Assert.assertEquals;
4+
5+
import java.util.Arrays;
6+
import java.util.List;
7+
8+
import org.junit.Test;
9+
import org.junit.runner.RunWith;
10+
import org.junit.runners.Parameterized;
11+
12+
@RunWith(Parameterized.class)
13+
public class ParameterizedTest
14+
{
15+
@Parameterized.Parameters
16+
public static List<Integer> parameters() throws Exception
17+
{
18+
return Arrays.asList( 0, 1 );
19+
}
20+
21+
@Parameterized.Parameter(0)
22+
public int expected;
23+
24+
@Test
25+
public void notFlaky()
26+
{
27+
assertEquals( expected, 0 );
28+
}
29+
}

‎surefire-its/src/test/resources/surefire-2065-common/src/test/java/pkg/junit4/ParameterizedWithDisplayNameTest.java

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
package pkg.junit4;
2+
3+
import static org.junit.Assert.assertEquals;
4+
5+
import java.util.Arrays;
6+
import java.util.List;
7+
8+
import org.junit.Test;
9+
import org.junit.runner.RunWith;
10+
import org.junit.runners.Parameterized;
11+
12+
@RunWith(Parameterized.class)
13+
public class ParameterizedWithDisplayNameTest
14+
{
15+
private static int count = 0;
16+
17+
@Parameterized.Parameters(name = "value={0}")
18+
public static List<Integer> parameters() throws Exception
19+
{
20+
return Arrays.asList( 0, 1 );
21+
}
22+
23+
@Parameterized.Parameter(0)
24+
public int expected;
25+
26+
@Test
27+
public void notFlaky()
28+
{
29+
assertEquals( expected, 0 );
30+
}
31+
}

‎surefire-its/src/test/resources/surefire-2065-common/src/test/java/pkg/junit5/ParameterizedTest.java

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package pkg.junit5;
2+
3+
import static org.junit.jupiter.api.Assertions.assertEquals;
4+
5+
import java.util.Arrays;
6+
import java.util.stream.Stream;
7+
8+
import org.junit.jupiter.api.extension.ExtensionContext;
9+
import org.junit.jupiter.params.provider.Arguments;
10+
import org.junit.jupiter.params.provider.ArgumentsProvider;
11+
import org.junit.jupiter.params.provider.ArgumentsSource;
12+
13+
class ParameterizedTest
14+
{
15+
static class ParameterSource implements ArgumentsProvider
16+
{
17+
@Override
18+
public Stream<? extends Arguments> provideArguments( ExtensionContext context ) throws Exception
19+
{
20+
return Arrays.asList(0, 1).stream().map( Arguments::of );
21+
}
22+
}
23+
24+
@org.junit.jupiter.params.ParameterizedTest
25+
@ArgumentsSource(value = ParameterSource.class)
26+
public void notFlaky(int expected)
27+
{
28+
assertEquals( expected, 0 );
29+
}
30+
}

‎surefire-its/src/test/resources/surefire-2065-common/src/test/java/pkg/junit5/ParameterizedWithDisplayNameTest.java

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package pkg.junit5;
2+
3+
import static org.junit.jupiter.api.Assertions.assertEquals;
4+
5+
import java.util.Arrays;
6+
import java.util.stream.Stream;
7+
8+
import org.junit.jupiter.api.extension.ExtensionContext;
9+
import org.junit.jupiter.params.provider.Arguments;
10+
import org.junit.jupiter.params.provider.ArgumentsProvider;
11+
import org.junit.jupiter.params.provider.ArgumentsSource;
12+
13+
class ParameterizedWithDisplayNameTest
14+
{
15+
static class ParameterSource implements ArgumentsProvider
16+
{
17+
@Override
18+
public Stream<? extends Arguments> provideArguments( ExtensionContext context ) throws Exception
19+
{
20+
return Arrays.asList(0, 1).stream().map( Arguments::of );
21+
}
22+
}
23+
24+
@org.junit.jupiter.params.ParameterizedTest(name = "value={0}")
25+
@ArgumentsSource(value = ParameterSource.class)
26+
public void notFlaky(int expected)
27+
{
28+
assertEquals( expected, 0 );
29+
}
30+
}

‎surefire-its/src/test/resources/surefire-2065-junit4/pom.xml

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one
4+
~ or more contributor license agreements. See the NOTICE file
5+
~ distributed with this work for additional information
6+
~ regarding copyright ownership. The ASF licenses this file
7+
~ to you under the Apache License, Version 2.0 (the
8+
~ "License"); you may not use this file except in compliance
9+
~ with the License. You may obtain a copy of the License at
10+
~
11+
~ http://www.apache.org/licenses/LICENSE-2.0
12+
~
13+
~ Unless required by applicable law or agreed to in writing,
14+
~ software distributed under the License is distributed on an
15+
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
~ KIND, either express or implied. See the License for the
17+
~ specific language governing permissions and limitations
18+
~ under the License.
19+
-->
20+
21+
<project xmlns="http://maven.apache.org/POM/4.0.0"
22+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
23+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
24+
<modelVersion>4.0.0</modelVersion>
25+
26+
<groupId>org.example</groupId>
27+
<artifactId>surefire-2065-junit4</artifactId>
28+
<version>1.0-SNAPSHOT</version>
29+
30+
<properties>
31+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
32+
<maven.compiler.source>${java.specification.version}</maven.compiler.source>
33+
<maven.compiler.target>${java.specification.version}</maven.compiler.target>
34+
<junit4.version>4.12</junit4.version>
35+
</properties>
36+
37+
<dependencies>
38+
<dependency>
39+
<groupId>junit</groupId>
40+
<artifactId>junit</artifactId>
41+
<version>${junit4.version}</version>
42+
<scope>test</scope>
43+
</dependency>
44+
</dependencies>
45+
46+
<build>
47+
<plugins>
48+
<plugin>
49+
<groupId>org.apache.maven.plugins</groupId>
50+
<artifactId>maven-surefire-plugin</artifactId>
51+
<version>${surefire.version}</version>
52+
<configuration>
53+
<rerunFailingTestsCount>1</rerunFailingTestsCount>
54+
</configuration>
55+
</plugin>
56+
</plugins>
57+
</build>
58+
59+
</project>

0 commit comments

Comments
 (0)