Skip to content

Commit aabe9c5

Browse files
katcharovkatcharov
committed
Retry flaky unified tests
JAVA-5393
1 parent 103815f commit aabe9c5

File tree

9 files changed

+157
-45
lines changed

9 files changed

+157
-45
lines changed

‎driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ internal class UnifiedCrudTest() : UnifiedTest() {
2424
@JvmStatic
2525
@Throws(URISyntaxException::class, IOException::class)
2626
fun data(): Collection<Arguments>? {
27-
return getTestData("unified-test-format/crud")
27+
return getTestData("unified-test-format/crud", true)
2828
}
2929
}
3030
}

‎driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ internal class UnifiedCrudTest() : UnifiedTest() {
2424
@JvmStatic
2525
@Throws(URISyntaxException::class, IOException::class)
2626
fun data(): Collection<Arguments>? {
27-
return getTestData("unified-test-format/crud")
27+
return getTestData("unified-test-format/crud", false)
2828
}
2929
}
3030
}

‎driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,18 +99,25 @@ The Reactive Streams specification prevents us from allowing a subsequent next c
9999
@MethodSource("data")
100100
@Override
101101
public void shouldPassAllOutcomes(
102+
final String testName,
102103
@Nullable final String fileDescription,
103104
@Nullable final String testDescription,
104105
@Nullable final String directoryName,
106+
final int attemptNumber,
107+
final int totalAttempts,
105108
final String schemaVersion,
106109
@Nullable final BsonArray runOnRequirements,
107110
final BsonArray entitiesArray,
108111
final BsonArray initialData,
109112
final BsonDocument definition) {
110113
try {
111-
super.shouldPassAllOutcomes(fileDescription,
114+
super.shouldPassAllOutcomes(
115+
testName,
116+
fileDescription,
112117
testDescription,
113118
directoryName,
119+
attemptNumber,
120+
totalAttempts,
114121
schemaVersion,
115122
runOnRequirements,
116123
entitiesArray,

‎driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,19 @@
2424
import com.mongodb.client.unified.UnifiedTest;
2525
import com.mongodb.client.unified.UnifiedTestModifications;
2626
import com.mongodb.client.vault.ClientEncryption;
27+
import com.mongodb.lang.NonNull;
2728
import com.mongodb.reactivestreams.client.MongoClients;
2829
import com.mongodb.reactivestreams.client.gridfs.GridFSBuckets;
2930
import com.mongodb.reactivestreams.client.internal.vault.ClientEncryptionImpl;
3031
import com.mongodb.reactivestreams.client.syncadapter.SyncClientEncryption;
3132
import com.mongodb.reactivestreams.client.syncadapter.SyncGridFSBucket;
3233
import com.mongodb.reactivestreams.client.syncadapter.SyncMongoClient;
3334
import com.mongodb.reactivestreams.client.syncadapter.SyncMongoDatabase;
35+
import org.junit.jupiter.params.provider.Arguments;
36+
37+
import java.io.IOException;
38+
import java.net.URISyntaxException;
39+
import java.util.Collection;
3440

3541
import static com.mongodb.client.unified.UnifiedTestModifications.Modifier;
3642
import static com.mongodb.client.unified.UnifiedTestModifications.TestDef;
@@ -94,4 +100,9 @@ protected void postCleanUp(final TestDef testDef) {
94100
disableSleep();
95101
}
96102
}
103+
104+
@NonNull
105+
protected static Collection<Arguments> getTestData(final String directory) throws URISyntaxException, IOException {
106+
return getTestData(directory, true);
107+
}
97108
}

‎driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,12 @@
2525
import com.mongodb.client.gridfs.GridFSBuckets;
2626
import com.mongodb.client.internal.ClientEncryptionImpl;
2727
import com.mongodb.client.vault.ClientEncryption;
28+
import com.mongodb.lang.NonNull;
29+
import org.junit.jupiter.params.provider.Arguments;
30+
31+
import java.io.IOException;
32+
import java.net.URISyntaxException;
33+
import java.util.Collection;
2834

2935
public abstract class UnifiedSyncTest extends UnifiedTest {
3036
protected UnifiedSyncTest() {
@@ -44,4 +50,9 @@ protected GridFSBucket createGridFSBucket(final MongoDatabase database) {
4450
protected ClientEncryption createClientEncryption(final MongoClient keyVaultClient, final ClientEncryptionSettings clientEncryptionSettings) {
4551
return new ClientEncryptionImpl(keyVaultClient, clientEncryptionSettings);
4652
}
53+
54+
@NonNull
55+
protected static Collection<Arguments> getTestData(final String directory) throws URISyntaxException, IOException {
56+
return getTestData(directory, false);
57+
}
4758
}

‎driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java

Lines changed: 90 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -21,27 +21,27 @@
2121
import com.mongodb.MongoNamespace;
2222
import com.mongodb.ReadPreference;
2323
import com.mongodb.UnixServerAddress;
24-
import com.mongodb.client.unified.UnifiedTestModifications.TestDef;
25-
import com.mongodb.event.TestServerMonitorListener;
26-
import com.mongodb.internal.logging.LogMessage;
27-
import com.mongodb.logging.TestLoggingInterceptor;
2824
import com.mongodb.WriteConcern;
2925
import com.mongodb.client.ClientSession;
3026
import com.mongodb.client.MongoClient;
3127
import com.mongodb.client.MongoDatabase;
3228
import com.mongodb.client.gridfs.GridFSBucket;
3329
import com.mongodb.client.model.Filters;
3430
import com.mongodb.client.test.CollectionHelper;
31+
import com.mongodb.client.unified.UnifiedTestModifications.TestDef;
3532
import com.mongodb.client.vault.ClientEncryption;
3633
import com.mongodb.connection.ClusterDescription;
3734
import com.mongodb.connection.ClusterType;
3835
import com.mongodb.connection.ServerDescription;
3936
import com.mongodb.event.CommandEvent;
4037
import com.mongodb.event.CommandStartedEvent;
38+
import com.mongodb.event.TestServerMonitorListener;
4139
import com.mongodb.internal.connection.TestCommandListener;
4240
import com.mongodb.internal.connection.TestConnectionPoolListener;
41+
import com.mongodb.internal.logging.LogMessage;
4342
import com.mongodb.lang.NonNull;
4443
import com.mongodb.lang.Nullable;
44+
import com.mongodb.logging.TestLoggingInterceptor;
4545
import com.mongodb.test.AfterBeforeParameterResolver;
4646
import org.bson.BsonArray;
4747
import org.bson.BsonBoolean;
@@ -57,14 +57,17 @@
5757
import org.junit.jupiter.params.ParameterizedTest;
5858
import org.junit.jupiter.params.provider.Arguments;
5959
import org.junit.jupiter.params.provider.MethodSource;
60+
import org.opentest4j.AssertionFailedError;
6061
import org.opentest4j.TestAbortedException;
6162

6263
import java.io.File;
6364
import java.io.IOException;
6465
import java.net.URISyntaxException;
66+
import java.text.MessageFormat;
6567
import java.util.ArrayList;
6668
import java.util.Collection;
6769
import java.util.Collections;
70+
import java.util.HashSet;
6871
import java.util.List;
6972
import java.util.Set;
7073
import java.util.concurrent.ExecutionException;
@@ -81,6 +84,7 @@
8184
import static com.mongodb.client.test.CollectionHelper.getCurrentClusterTime;
8285
import static com.mongodb.client.test.CollectionHelper.killAllSessions;
8386
import static com.mongodb.client.unified.RunOnRequirementsMatcher.runOnRequirementsMet;
87+
import static com.mongodb.client.unified.UnifiedTestModifications.doSkips;
8488
import static com.mongodb.client.unified.UnifiedTestModifications.testDef;
8589
import static java.util.Collections.singletonList;
8690
import static java.util.stream.Collectors.toList;
@@ -101,6 +105,9 @@ public abstract class UnifiedTest {
101105
private static final Set<String> PRESTART_POOL_ASYNC_WORK_MANAGER_FILE_DESCRIPTIONS = Collections.singleton(
102106
"wait queue timeout errors include details about checked out connections");
103107

108+
public static final int ATTEMPTS = 3;
109+
private static Set<String> completed = new HashSet<>();
110+
104111
@Nullable
105112
private String fileDescription;
106113
private String schemaVersion;
@@ -155,32 +162,51 @@ public Entities getEntities() {
155162
}
156163

157164
@NonNull
158-
protected static Collection<Arguments> getTestData(final String directory) throws URISyntaxException, IOException {
165+
protected static Collection<Arguments> getTestData(final String directory, final boolean isReactive)
166+
throws URISyntaxException, IOException {
159167
List<Arguments> data = new ArrayList<>();
160168
for (File file : getTestFiles("/" + directory + "/")) {
161169
BsonDocument fileDocument = getTestDocument(file);
162-
163170
for (BsonValue cur : fileDocument.getArray("tests")) {
164-
data.add(UnifiedTest.createTestData(directory, fileDocument, cur.asDocument()));
171+
172+
final BsonDocument testDocument = cur.asDocument();
173+
String testDescription = testDocument.getString("description").getValue();
174+
String fileDescription = fileDocument.getString("description").getValue();
175+
TestDef testDef = testDef(directory, fileDescription, testDescription, isReactive);
176+
doSkips(testDef);
177+
178+
boolean forceFlaky = testDef.wasAssignedModifier(UnifiedTestModifications.Modifier.FORCE_FLAKY);
179+
boolean retry = forceFlaky || testDef.wasAssignedModifier(UnifiedTestModifications.Modifier.RETRY);
180+
181+
int attempts = retry ? ATTEMPTS : 1;
182+
if (forceFlaky) {
183+
attempts = 10;
184+
}
185+
186+
for (int attempt = 1; attempt <= attempts; attempt++) {
187+
String testName = !retry
188+
? MessageFormat.format("{0}: {1}", fileDescription, testDescription)
189+
: MessageFormat.format(
190+
"{0}: {1} ({2} of {3})",
191+
fileDescription, testDescription, attempt, attempts);
192+
data.add(Arguments.of(
193+
testName,
194+
fileDescription,
195+
testDescription,
196+
directory,
197+
attempt,
198+
attempts * (forceFlaky ? -1 : 1),
199+
fileDocument.getString("schemaVersion").getValue(),
200+
fileDocument.getArray("runOnRequirements", null),
201+
fileDocument.getArray("createEntities", new BsonArray()),
202+
fileDocument.getArray("initialData", new BsonArray()),
203+
testDocument));
204+
}
165205
}
166206
}
167207
return data;
168208
}
169209

170-
@NonNull
171-
private static Arguments createTestData(
172-
final String directory, final BsonDocument fileDocument, final BsonDocument testDocument) {
173-
return Arguments.of(
174-
fileDocument.getString("description").getValue(),
175-
testDocument.getString("description").getValue(),
176-
directory,
177-
fileDocument.getString("schemaVersion").getValue(),
178-
fileDocument.getArray("runOnRequirements", null),
179-
fileDocument.getArray("createEntities", new BsonArray()),
180-
fileDocument.getArray("initialData", new BsonArray()),
181-
testDocument);
182-
}
183-
184210
protected BsonDocument getDefinition() {
185211
return definition;
186212
}
@@ -193,9 +219,12 @@ protected BsonDocument getDefinition() {
193219

194220
@BeforeEach
195221
public void setUp(
222+
final String testName,
196223
@Nullable final String fileDescription,
197224
@Nullable final String testDescription,
198225
@Nullable final String directoryName,
226+
final int attemptNumber,
227+
final int totalAttempts,
199228
final String schemaVersion,
200229
@Nullable final BsonArray runOnRequirements,
201230
final BsonArray entitiesArray,
@@ -288,8 +317,9 @@ protected void postCleanUp(final TestDef testDef) {
288317
}
289318

290319
/**
291-
* This method is called once per {@link #setUp(String, String, String, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)},
292-
* unless {@link #setUp(String, String, String, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)} fails unexpectedly.
320+
* This method is called once per
321+
* {@link #setUp(String, String, String, String, int, int, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)}, unless
322+
* {@link #setUp(String, String, String, String, int, int, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)} fails unexpectedly.
293323
*/
294324
protected void skips(final String fileDescription, final String testDescription) {
295325
}
@@ -298,40 +328,58 @@ protected boolean isReactive() {
298328
return false;
299329
}
300330

301-
@ParameterizedTest(name = "{0}: {1}")
331+
@ParameterizedTest(name = "{0}")
302332
@MethodSource("data")
303333
public void shouldPassAllOutcomes(
334+
final String testName,
304335
@Nullable final String fileDescription,
305336
@Nullable final String testDescription,
306337
@Nullable final String directoryName,
338+
final int attemptNumber,
339+
final int totalAttempts,
307340
final String schemaVersion,
308341
@Nullable final BsonArray runOnRequirements,
309342
final BsonArray entitiesArray,
310343
final BsonArray initialData,
311344
final BsonDocument definition) {
312-
BsonArray operations = definition.getArray("operations");
313-
for (int i = 0; i < operations.size(); i++) {
314-
BsonValue cur = operations.get(i);
315-
assertOperation(rootContext, cur.asDocument(), i);
345+
boolean forceFlaky = totalAttempts < 0;
346+
if (!forceFlaky) {
347+
assumeFalse(completed.contains(testName), "Skipping retryable test that succeeded");
348+
completed.add(testName);
316349
}
350+
try {
351+
BsonArray operations = definition.getArray("operations");
352+
for (int i = 0; i < operations.size(); i++) {
353+
BsonValue cur = operations.get(i);
354+
assertOperation(rootContext, cur.asDocument(), i);
355+
}
317356

318-
if (definition.containsKey("outcome")) {
319-
assertOutcome(rootContext);
320-
}
357+
if (definition.containsKey("outcome")) {
358+
assertOutcome(rootContext);
359+
}
321360

322-
if (definition.containsKey("expectEvents")) {
323-
compareEvents(rootContext, definition);
324-
}
361+
if (definition.containsKey("expectEvents")) {
362+
compareEvents(rootContext, definition);
363+
}
325364

326-
if (definition.containsKey("expectLogMessages")) {
327-
ArrayList<LogMatcher.Tweak> tweaks = new ArrayList<>(singletonList(
328-
// `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value
329-
LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION)));
330-
if (getMongoClientSettings().getClusterSettings()
331-
.getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) {
332-
tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT));
365+
if (definition.containsKey("expectLogMessages")) {
366+
ArrayList<LogMatcher.Tweak> tweaks = new ArrayList<>(singletonList(
367+
// `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value
368+
LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION)));
369+
if (getMongoClientSettings().getClusterSettings()
370+
.getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) {
371+
tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT));
372+
}
373+
compareLogMessages(rootContext, definition, tweaks);
374+
}
375+
} catch (AssertionFailedError e) {
376+
completed.remove(testName);
377+
boolean lastAttempt = attemptNumber == Math.abs(totalAttempts);
378+
if (forceFlaky || lastAttempt) {
379+
throw e;
380+
} else {
381+
assumeFalse(completed.contains(testName), "Ignoring failure and retrying attempt " + attemptNumber);
333382
}
334-
compareLogMessages(rootContext, definition, tweaks);
335383
}
336384
}
337385

‎driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestFailureValidator.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,19 +36,25 @@ final class UnifiedTestFailureValidator extends UnifiedSyncTest {
3636
@Override
3737
@BeforeEach
3838
public void setUp(
39+
final String testName,
3940
@Nullable final String fileDescription,
4041
@Nullable final String testDescription,
4142
final String directoryName,
43+
final int attemptNumber,
44+
final int totalAttempts,
4245
final String schemaVersion,
4346
@Nullable final BsonArray runOnRequirements,
4447
final BsonArray entitiesArray,
4548
final BsonArray initialData,
4649
final BsonDocument definition) {
4750
try {
4851
super.setUp(
52+
testName,
4953
fileDescription,
5054
testDescription,
5155
directoryName,
56+
attemptNumber,
57+
totalAttempts,
5258
schemaVersion,
5359
runOnRequirements,
5460
entitiesArray,
@@ -63,9 +69,12 @@ public void setUp(
6369
@ParameterizedTest
6470
@MethodSource("data")
6571
public void shouldPassAllOutcomes(
72+
final String testName,
6673
@Nullable final String fileDescription,
6774
@Nullable final String testDescription,
6875
@Nullable final String directoryName,
76+
final int attemptNumber,
77+
final int totalAttempts,
6978
final String schemaVersion,
7079
@Nullable final BsonArray runOnRequirements,
7180
final BsonArray entitiesArray,
@@ -74,9 +83,12 @@ public void shouldPassAllOutcomes(
7483
if (exception == null) {
7584
try {
7685
super.shouldPassAllOutcomes(
86+
testName,
7787
fileDescription,
7888
testDescription,
7989
directoryName,
90+
attemptNumber,
91+
totalAttempts,
8092
schemaVersion,
8193
runOnRequirements,
8294
entitiesArray,

0 commit comments

Comments
 (0)