diff --git a/driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt b/driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt index 036ec5afcc..5091058573 100644 --- a/driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt +++ b/driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt @@ -24,7 +24,7 @@ internal class UnifiedCrudTest() : UnifiedTest() { @JvmStatic @Throws(URISyntaxException::class, IOException::class) fun data(): Collection? { - return getTestData("unified-test-format/crud") + return getTestData("unified-test-format/crud", true) } } } diff --git a/driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt b/driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt index eb06f5c187..f030cb5464 100644 --- a/driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt +++ b/driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt @@ -24,7 +24,7 @@ internal class UnifiedCrudTest() : UnifiedTest() { @JvmStatic @Throws(URISyntaxException::class, IOException::class) fun data(): Collection? { - return getTestData("unified-test-format/crud") + return getTestData("unified-test-format/crud", false) } } } diff --git a/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java b/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java index 168ff4b8f8..a1063f0536 100644 --- a/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java +++ b/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java @@ -99,18 +99,25 @@ The Reactive Streams specification prevents us from allowing a subsequent next c @MethodSource("data") @Override public void shouldPassAllOutcomes( + final String testName, @Nullable final String fileDescription, @Nullable final String testDescription, @Nullable final String directoryName, + final int attemptNumber, + final int totalAttempts, final String schemaVersion, @Nullable final BsonArray runOnRequirements, final BsonArray entitiesArray, final BsonArray initialData, final BsonDocument definition) { try { - super.shouldPassAllOutcomes(fileDescription, + super.shouldPassAllOutcomes( + testName, + fileDescription, testDescription, directoryName, + attemptNumber, + totalAttempts, schemaVersion, runOnRequirements, entitiesArray, diff --git a/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java b/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java index 62c1315e24..28c8a27f8f 100644 --- a/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java +++ b/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java @@ -24,6 +24,7 @@ import com.mongodb.client.unified.UnifiedTest; import com.mongodb.client.unified.UnifiedTestModifications; import com.mongodb.client.vault.ClientEncryption; +import com.mongodb.lang.NonNull; import com.mongodb.reactivestreams.client.MongoClients; import com.mongodb.reactivestreams.client.gridfs.GridFSBuckets; import com.mongodb.reactivestreams.client.internal.vault.ClientEncryptionImpl; @@ -31,6 +32,11 @@ import com.mongodb.reactivestreams.client.syncadapter.SyncGridFSBucket; import com.mongodb.reactivestreams.client.syncadapter.SyncMongoClient; import com.mongodb.reactivestreams.client.syncadapter.SyncMongoDatabase; +import org.junit.jupiter.params.provider.Arguments; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Collection; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier; import static com.mongodb.client.unified.UnifiedTestModifications.TestDef; @@ -94,4 +100,9 @@ protected void postCleanUp(final TestDef testDef) { disableSleep(); } } + + @NonNull + protected static Collection getTestData(final String directory) throws URISyntaxException, IOException { + return getTestData(directory, true); + } } diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java index 37db7cfe90..afcc8e4f1a 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java @@ -25,6 +25,12 @@ import com.mongodb.client.gridfs.GridFSBuckets; import com.mongodb.client.internal.ClientEncryptionImpl; import com.mongodb.client.vault.ClientEncryption; +import com.mongodb.lang.NonNull; +import org.junit.jupiter.params.provider.Arguments; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Collection; public abstract class UnifiedSyncTest extends UnifiedTest { protected UnifiedSyncTest() { @@ -44,4 +50,9 @@ protected GridFSBucket createGridFSBucket(final MongoDatabase database) { protected ClientEncryption createClientEncryption(final MongoClient keyVaultClient, final ClientEncryptionSettings clientEncryptionSettings) { return new ClientEncryptionImpl(keyVaultClient, clientEncryptionSettings); } + + @NonNull + protected static Collection getTestData(final String directory) throws URISyntaxException, IOException { + return getTestData(directory, false); + } } diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java index d1d1fc58c8..1ff405c9e1 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java @@ -21,10 +21,6 @@ import com.mongodb.MongoNamespace; import com.mongodb.ReadPreference; import com.mongodb.UnixServerAddress; -import com.mongodb.client.unified.UnifiedTestModifications.TestDef; -import com.mongodb.event.TestServerMonitorListener; -import com.mongodb.internal.logging.LogMessage; -import com.mongodb.logging.TestLoggingInterceptor; import com.mongodb.WriteConcern; import com.mongodb.client.ClientSession; import com.mongodb.client.MongoClient; @@ -32,16 +28,20 @@ import com.mongodb.client.gridfs.GridFSBucket; import com.mongodb.client.model.Filters; import com.mongodb.client.test.CollectionHelper; +import com.mongodb.client.unified.UnifiedTestModifications.TestDef; import com.mongodb.client.vault.ClientEncryption; import com.mongodb.connection.ClusterDescription; import com.mongodb.connection.ClusterType; import com.mongodb.connection.ServerDescription; import com.mongodb.event.CommandEvent; import com.mongodb.event.CommandStartedEvent; +import com.mongodb.event.TestServerMonitorListener; import com.mongodb.internal.connection.TestCommandListener; import com.mongodb.internal.connection.TestConnectionPoolListener; +import com.mongodb.internal.logging.LogMessage; import com.mongodb.lang.NonNull; import com.mongodb.lang.Nullable; +import com.mongodb.logging.TestLoggingInterceptor; import com.mongodb.test.AfterBeforeParameterResolver; import org.bson.BsonArray; import org.bson.BsonBoolean; @@ -57,14 +57,17 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.opentest4j.AssertionFailedError; import org.opentest4j.TestAbortedException; import java.io.File; import java.io.IOException; import java.net.URISyntaxException; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -81,6 +84,7 @@ import static com.mongodb.client.test.CollectionHelper.getCurrentClusterTime; import static com.mongodb.client.test.CollectionHelper.killAllSessions; import static com.mongodb.client.unified.RunOnRequirementsMatcher.runOnRequirementsMet; +import static com.mongodb.client.unified.UnifiedTestModifications.doSkips; import static com.mongodb.client.unified.UnifiedTestModifications.testDef; import static java.util.Collections.singletonList; import static java.util.stream.Collectors.toList; @@ -91,6 +95,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeFalse; import static org.junit.jupiter.api.Assumptions.assumeTrue; import static util.JsonPoweredTestHelper.getTestDocument; import static util.JsonPoweredTestHelper.getTestFiles; @@ -100,6 +105,9 @@ public abstract class UnifiedTest { private static final Set PRESTART_POOL_ASYNC_WORK_MANAGER_FILE_DESCRIPTIONS = Collections.singleton( "wait queue timeout errors include details about checked out connections"); + public static final int ATTEMPTS = 3; + private static Set completed = new HashSet<>(); + @Nullable private String fileDescription; private String schemaVersion; @@ -154,32 +162,47 @@ public Entities getEntities() { } @NonNull - protected static Collection getTestData(final String directory) throws URISyntaxException, IOException { + protected static Collection getTestData(final String directory, final boolean isReactive) + throws URISyntaxException, IOException { List data = new ArrayList<>(); for (File file : getTestFiles("/" + directory + "/")) { BsonDocument fileDocument = getTestDocument(file); - for (BsonValue cur : fileDocument.getArray("tests")) { - data.add(UnifiedTest.createTestData(directory, fileDocument, cur.asDocument())); + + final BsonDocument testDocument = cur.asDocument(); + String testDescription = testDocument.getString("description").getValue(); + String fileDescription = fileDocument.getString("description").getValue(); + TestDef testDef = testDef(directory, fileDescription, testDescription, isReactive); + doSkips(testDef); + + boolean retry = testDef.wasAssignedModifier(UnifiedTestModifications.Modifier.RETRY); + int attempts = retry ? ATTEMPTS : 1; + + + for (int attempt = 1; attempt <= attempts; attempt++) { + String testName = !retry + ? MessageFormat.format("{0}: {1}", fileDescription, testDescription) + : MessageFormat.format( + "{0}: {1} ({2} of {3})", + fileDescription, testDescription, attempt, attempts); + data.add(Arguments.of( + testName, + fileDescription, + testDescription, + directory, + attempt, + attempts, + fileDocument.getString("schemaVersion").getValue(), + fileDocument.getArray("runOnRequirements", null), + fileDocument.getArray("createEntities", new BsonArray()), + fileDocument.getArray("initialData", new BsonArray()), + testDocument)); + } } } return data; } - @NonNull - private static Arguments createTestData( - final String directory, final BsonDocument fileDocument, final BsonDocument testDocument) { - return Arguments.of( - fileDocument.getString("description").getValue(), - testDocument.getString("description").getValue(), - directory, - fileDocument.getString("schemaVersion").getValue(), - fileDocument.getArray("runOnRequirements", null), - fileDocument.getArray("createEntities", new BsonArray()), - fileDocument.getArray("initialData", new BsonArray()), - testDocument); - } - protected BsonDocument getDefinition() { return definition; } @@ -192,9 +215,12 @@ protected BsonDocument getDefinition() { @BeforeEach public void setUp( + final String testName, @Nullable final String fileDescription, @Nullable final String testDescription, @Nullable final String directoryName, + final int attemptNumber, + final int totalAttempts, final String schemaVersion, @Nullable final BsonArray runOnRequirements, final BsonArray entitiesArray, @@ -216,7 +242,11 @@ public void setUp( ignoreExtraEvents = false; testDef = testDef(directoryName, fileDescription, testDescription, isReactive()); UnifiedTestModifications.doSkips(testDef); + + boolean skip = testDef.wasAssignedModifier(UnifiedTestModifications.Modifier.SKIP); + assumeFalse(skip, "Skipping test"); skips(fileDescription, testDescription); + assertTrue( schemaVersion.equals("1.0") || schemaVersion.equals("1.1") @@ -283,8 +313,9 @@ protected void postCleanUp(final TestDef testDef) { } /** - * This method is called once per {@link #setUp(String, String, String, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)}, - * unless {@link #setUp(String, String, String, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)} fails unexpectedly. + * This method is called once per + * {@link #setUp(String, String, String, String, int, int, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)}, unless + * {@link #setUp(String, String, String, String, int, int, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)} fails unexpectedly. */ protected void skips(final String fileDescription, final String testDescription) { } @@ -293,40 +324,54 @@ protected boolean isReactive() { return false; } - @ParameterizedTest(name = "{0}: {1}") + @ParameterizedTest(name = "{0}") @MethodSource("data") public void shouldPassAllOutcomes( + final String testName, @Nullable final String fileDescription, @Nullable final String testDescription, @Nullable final String directoryName, + final int attemptNumber, + final int totalAttempts, final String schemaVersion, @Nullable final BsonArray runOnRequirements, final BsonArray entitiesArray, final BsonArray initialData, final BsonDocument definition) { - BsonArray operations = definition.getArray("operations"); - for (int i = 0; i < operations.size(); i++) { - BsonValue cur = operations.get(i); - assertOperation(rootContext, cur.asDocument(), i); - } + assumeFalse(completed.contains(testName), "Skipping test already performed"); + completed.add(testName); + try { + BsonArray operations = definition.getArray("operations"); + for (int i = 0; i < operations.size(); i++) { + BsonValue cur = operations.get(i); + assertOperation(rootContext, cur.asDocument(), i); + } - if (definition.containsKey("outcome")) { - assertOutcome(rootContext); - } + if (definition.containsKey("outcome")) { + assertOutcome(rootContext); + } - if (definition.containsKey("expectEvents")) { - compareEvents(rootContext, definition); - } + if (definition.containsKey("expectEvents")) { + compareEvents(rootContext, definition); + } - if (definition.containsKey("expectLogMessages")) { - ArrayList tweaks = new ArrayList<>(singletonList( - // `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value - LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION))); - if (getMongoClientSettings().getClusterSettings() - .getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) { - tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT)); + if (definition.containsKey("expectLogMessages")) { + ArrayList tweaks = new ArrayList<>(singletonList( + // `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value + LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION))); + if (getMongoClientSettings().getClusterSettings() + .getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) { + tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT)); + } + compareLogMessages(rootContext, definition, tweaks); + } + } catch (AssertionFailedError e) { + completed.remove(testName); + if (attemptNumber == totalAttempts) { // last attempt + throw e; + } else { + assumeFalse(completed.contains(testName), "Ignoring failure and retrying attempt " + attemptNumber); } - compareLogMessages(rootContext, definition, tweaks); } } diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestFailureValidator.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestFailureValidator.java index b5bf53f4b8..0472ef8e6c 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestFailureValidator.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestFailureValidator.java @@ -36,9 +36,12 @@ final class UnifiedTestFailureValidator extends UnifiedSyncTest { @Override @BeforeEach public void setUp( - final String directoryName, + final String testName, @Nullable final String fileDescription, @Nullable final String testDescription, + final String directoryName, + final int attemptNumber, + final int totalAttempts, final String schemaVersion, @Nullable final BsonArray runOnRequirements, final BsonArray entitiesArray, @@ -46,9 +49,12 @@ public void setUp( final BsonDocument definition) { try { super.setUp( - directoryName, + testName, fileDescription, testDescription, + directoryName, + attemptNumber, + totalAttempts, schemaVersion, runOnRequirements, entitiesArray, @@ -63,9 +69,12 @@ public void setUp( @ParameterizedTest @MethodSource("data") public void shouldPassAllOutcomes( + final String testName, @Nullable final String fileDescription, @Nullable final String testDescription, @Nullable final String directoryName, + final int attemptNumber, + final int totalAttempts, final String schemaVersion, @Nullable final BsonArray runOnRequirements, final BsonArray entitiesArray, @@ -74,9 +83,12 @@ public void shouldPassAllOutcomes( if (exception == null) { try { super.shouldPassAllOutcomes( + testName, fileDescription, testDescription, directoryName, + attemptNumber, + totalAttempts, schemaVersion, runOnRequirements, entitiesArray, diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java index ea8180ead0..e54052088c 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java @@ -18,7 +18,6 @@ import com.mongodb.assertions.Assertions; import com.mongodb.lang.NonNull; -import com.mongodb.lang.Nullable; import java.util.ArrayList; import java.util.Arrays; @@ -31,14 +30,17 @@ import static com.mongodb.ClusterFixture.isSharded; import static com.mongodb.ClusterFixture.serverVersionLessThan; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.IGNORE_EXTRA_EVENTS; +import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.RETRY; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.SLEEP_AFTER_CURSOR_CLOSE; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.SLEEP_AFTER_CURSOR_OPEN; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.WAIT_FOR_BATCH_CURSOR_CREATION; -import static org.junit.jupiter.api.Assumptions.assumeFalse; public final class UnifiedTestModifications { public static void doSkips(final TestDef def) { + def.modify(RETRY) + .test("client-side-encryption", "namedKMS-rewrapManyDataKey", "rewrap to kmip:name1"); + // atlas-data-lake def.skipAccordingToSpec("Data lake tests should only run on data lake") @@ -360,10 +362,6 @@ public boolean wasAssignedModifier(final Modifier modifier) { */ public static final class TestApplicator { private final TestDef testDef; - - private final boolean shouldSkip; - @Nullable - private final String reasonToApply; private final List modifiersToApply; private Supplier precondition; private boolean matchWasPerformed = false; @@ -372,19 +370,15 @@ private TestApplicator( final TestDef testDef, final List modifiersToApply) { this.testDef = testDef; - this.shouldSkip = false; - this.reasonToApply = null; this.modifiersToApply = modifiersToApply; } private TestApplicator( final TestDef testDef, @NonNull - final String reason) { + final String skipReason) { this.testDef = testDef; - this.shouldSkip = true; - this.reasonToApply = reason; - this.modifiersToApply = new ArrayList<>(); + this.modifiersToApply = Arrays.asList(Modifier.SKIP); } private TestApplicator onMatch(final boolean match) { @@ -392,12 +386,8 @@ private TestApplicator onMatch(final boolean match) { if (precondition != null && !precondition.get()) { return this; } - if (shouldSkip) { - assumeFalse(match, reasonToApply); - } else { - if (match) { - this.testDef.modifiers.addAll(this.modifiersToApply); - } + if (match) { + this.testDef.modifiers.addAll(this.modifiersToApply); } return this; } @@ -510,5 +500,13 @@ public enum Modifier { * Reactive only. */ WAIT_FOR_BATCH_CURSOR_CREATION, + /** + * Retry the test on failure. + */ + RETRY, + /** + * Skip the test. + */ + SKIP, } } diff --git a/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java b/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java index 0e995cb34f..7aba736aeb 100644 --- a/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java +++ b/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java @@ -98,18 +98,24 @@ protected boolean terminateLoop() { BsonArray createEntities = fileDocument.getArray("createEntities", new BsonArray()); BsonArray initialData = fileDocument.getArray("initialData", new BsonArray()); unifiedTest.setUp( + "", null, null, null, + 1, + 1, schemaVersion, runOnRequirements, createEntities, initialData, testDocument); unifiedTest.shouldPassAllOutcomes( + "", null, null, null, + 1, + 1, schemaVersion, runOnRequirements, createEntities,