From a6e18f912a25aa5a620655b449bc40b37f9e9557 Mon Sep 17 00:00:00 2001 From: Maxim Katcharov Date: Tue, 28 Jan 2025 08:47:51 -0700 Subject: [PATCH 1/5] Add search integration tests --- .evergreen/.evg.yml | 76 +++--- .evergreen/run-atlas-search-tests.sh | 4 +- ...atesBinaryVectorSearchIntegrationTest.java | 2 +- .../AggregatesSearchIntegrationTest.java | 3 + .../com/mongodb/client/DatabaseTestCase.java | 23 +- .../com/mongodb/client/Fixture.java | 45 ++-- .../model/mql/AggregatesSearchTest.java | 249 ++++++++++++++++++ 7 files changed, 316 insertions(+), 86 deletions(-) create mode 100644 driver-sync/src/test/functional/com/mongodb/client/model/mql/AggregatesSearchTest.java diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index a57f6473b6f..4ec4fdcf8c8 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -509,6 +509,19 @@ functions: mongo --nodb setup.js aws_e2e_ecs.js cd - + run-atlas-connectivity-test-func: + - command: shell.exec + type: test + params: + silent: true + working_dir: "src" + script: | + # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does) + # The connection strings are pipe-delimited + MONGODB_URIS="${atlas_free_tier_uri}|${atlas_replica_set_uri}|${atlas_sharded_uri}|${atlas_tls_v11_uri}|${atlas_tls_v12_uri}|${atlas_free_tier_uri_srv}|${atlas_replica_set_uri_srv}|${atlas_sharded_uri_srv}|${atlas_tls_v11_uri_srv}|${atlas_tls_v12_uri_srv}|${atlas_serverless_uri}|${atlas_serverless_uri_srv}" \ + JAVA_VERSION="8" \ + .evergreen/run-connectivity-tests.sh + "run atlas data lake test": - command: shell.exec type: test @@ -627,19 +640,6 @@ functions: ${PREPARE_SHELL} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} JAVA_VERSION=${JAVA_VERSION} TOPOLOGY=${TOPOLOGY} STORAGE_ENGINE=${STORAGE_ENGINE} MONGODB_URI="${MONGODB_URI}" .evergreen/run-mmapv1-storage-test.sh - "run atlas test": - - command: shell.exec - type: test - params: - silent: true - working_dir: "src" - script: | - # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does) - # The connection strings are pipe-delimited - JAVA_VERSION="8" \ - MONGODB_URIS="${atlas_free_tier_uri}|${atlas_replica_set_uri}|${atlas_sharded_uri}|${atlas_tls_v11_uri}|${atlas_tls_v12_uri}|${atlas_free_tier_uri_srv}|${atlas_replica_set_uri_srv}|${atlas_sharded_uri_srv}|${atlas_tls_v11_uri_srv}|${atlas_tls_v12_uri_srv}|${atlas_serverless_uri}|${atlas_serverless_uri_srv}" \ - .evergreen/run-connectivity-tests.sh - run socks5 tests: - command: shell.exec type: test @@ -1471,6 +1471,20 @@ tasks: commands: - func: "run atlas search test" + - name: "atlas-test-task" + commands: + - func: run-atlas-connectivity-test-func + + - name: "test-atlas-search-index-helpers" + commands: + - command: subprocess.exec + params: + working_dir: src + binary: bash + add_expansions_to_env: true + args: + - .evergreen/run-atlas-search-index-management-tests.sh + - name: "gssapi-auth-test" commands: - func: "run gssapi auth test" @@ -1490,20 +1504,6 @@ tasks: - func: "bootstrap mongo-orchestration" - func: "run netty tests" - - name: "atlas-test" - commands: - - func: "run atlas test" - - - name: "test-atlas-search-index-helpers" - commands: - - command: subprocess.exec - params: - working_dir: src - binary: bash - add_expansions_to_env: true - args: - - .evergreen/run-atlas-search-index-management-tests.sh - - name: publish-snapshot depends_on: - variant: "static-checks" @@ -1630,7 +1630,6 @@ tasks: echo "Untarring file ... begin" GCPKMS_CMD="tar xf mongo-java-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Untarring file ... end" - - command: shell.exec type: test params: @@ -1918,7 +1917,7 @@ axes: batchtime: 10080 # 7 days task_groups: - - name: test_atlas_task_group_search_indexes + - name: test-atlas-search-task-group setup_group: - func: fetch source - func: prepare resources @@ -1944,6 +1943,7 @@ task_groups: setup_group_timeout_secs: 1800 tasks: - test-atlas-search-index-helpers + - name: testgcpkms_task_group setup_group_can_fail_task: true setup_group_timeout_secs: 1800 # 30 minutes @@ -1979,6 +1979,7 @@ task_groups: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh tasks: - testgcpkms-task + - name: testazurekms_task_group setup_group_can_fail_task: true setup_group_timeout_secs: 1800 # 30 minutes @@ -2023,6 +2024,7 @@ task_groups: $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh tasks: - testazurekms-task + - name: test_atlas_task_group setup_group: - func: fetch source @@ -2295,12 +2297,6 @@ buildvariants: tasks: - name: "plain-auth-test" -- name: rhel80-test-search-indexes - display_name: Atlas Search Index Management Tests - run_on: rhel80-small - tasks: - - name: "test_atlas_task_group_search_indexes" - - name: "oidc-auth-test" display_name: "OIDC Auth" run_on: ubuntu2204-small @@ -2361,11 +2357,11 @@ buildvariants: tasks: - name: "atlas-data-lake-test" -- name: atlas-test +- name: atlas-test-variant display_name: "Atlas test" run_on: rhel80-small tasks: - - name: "atlas-test" + - name: atlas-test-task - name: atlas-search-test display_name: "Atlas Search test" @@ -2373,6 +2369,12 @@ buildvariants: tasks: - name: "atlas-search-test" +- name: atlas-search-variant + display_name: "Atlas Search Tests" + run_on: rhel80-small + tasks: + - name: "test-atlas-search-task-group" + - name: "reactive-streams-tck-test" display_name: "Reactive Streams TCK tests" run_on: rhel80-small diff --git a/.evergreen/run-atlas-search-tests.sh b/.evergreen/run-atlas-search-tests.sh index 36cc981b3f4..f207647825f 100755 --- a/.evergreen/run-atlas-search-tests.sh +++ b/.evergreen/run-atlas-search-tests.sh @@ -16,4 +16,6 @@ echo "Running Atlas Search tests" ./gradlew --stacktrace --info \ -Dorg.mongodb.test.atlas.search=true \ -Dorg.mongodb.test.uri=${MONGODB_URI} \ - driver-core:test --tests AggregatesSearchIntegrationTest --tests AggregatesVectorSearchIntegrationTest + driver-core:test --tests AggregatesSearchIntegrationTest \ + --tests AggregatesBinaryVectorSearchIntegrationTest \ + --tests AggregatesSearchTest \ diff --git a/driver-core/src/test/functional/com/mongodb/client/model/search/AggregatesBinaryVectorSearchIntegrationTest.java b/driver-core/src/test/functional/com/mongodb/client/model/search/AggregatesBinaryVectorSearchIntegrationTest.java index 0d5aad1085a..a242367992f 100644 --- a/driver-core/src/test/functional/com/mongodb/client/model/search/AggregatesBinaryVectorSearchIntegrationTest.java +++ b/driver-core/src/test/functional/com/mongodb/client/model/search/AggregatesBinaryVectorSearchIntegrationTest.java @@ -22,9 +22,9 @@ import com.mongodb.client.model.SearchIndexType; import com.mongodb.client.test.CollectionHelper; import com.mongodb.internal.operation.SearchIndexRequest; +import org.bson.BinaryVector; import org.bson.BsonDocument; import org.bson.Document; -import org.bson.BinaryVector; import org.bson.codecs.DocumentCodec; import org.bson.conversions.Bson; import org.junit.jupiter.api.AfterAll; diff --git a/driver-core/src/test/functional/com/mongodb/client/model/search/AggregatesSearchIntegrationTest.java b/driver-core/src/test/functional/com/mongodb/client/model/search/AggregatesSearchIntegrationTest.java index d90f4a0575c..be87bca82f6 100644 --- a/driver-core/src/test/functional/com/mongodb/client/model/search/AggregatesSearchIntegrationTest.java +++ b/driver-core/src/test/functional/com/mongodb/client/model/search/AggregatesSearchIntegrationTest.java @@ -121,6 +121,9 @@ import static org.junit.jupiter.params.provider.Arguments.arguments; /** + * Use this class when needing to test against MFLIX specifically. Otherwise, + * see AggregatesSearchTest. + *

* These tests require the sample data * and the following Atlas Search indices: * diff --git a/driver-sync/src/test/functional/com/mongodb/client/DatabaseTestCase.java b/driver-sync/src/test/functional/com/mongodb/client/DatabaseTestCase.java index f2f1c5382cd..70479c4670b 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/DatabaseTestCase.java +++ b/driver-sync/src/test/functional/com/mongodb/client/DatabaseTestCase.java @@ -16,11 +16,8 @@ package com.mongodb.client; -import com.mongodb.MongoNamespace; import com.mongodb.client.test.CollectionHelper; import com.mongodb.internal.connection.ServerHelper; -import org.bson.BsonDocument; -import org.bson.BsonDocumentWrapper; import org.bson.Document; import org.bson.codecs.DocumentCodec; import org.junit.jupiter.api.AfterEach; @@ -40,7 +37,7 @@ public class DatabaseTestCase { @BeforeEach public void setUp() { - client = getMongoClient(); + client = getMongoClient(); database = client.getDatabase(getDefaultDatabaseName()); collection = database.getCollection(getClass().getName()); collection.drop(); @@ -58,23 +55,7 @@ public void tearDown() { } } - protected String getDatabaseName() { - return database.getName(); - } - - protected String getCollectionName() { - return collection.getNamespace().getCollectionName(); - } - - protected MongoNamespace getNamespace() { - return collection.getNamespace(); - } - protected CollectionHelper getCollectionHelper() { - return new CollectionHelper<>(new DocumentCodec(), getNamespace()); - } - - protected BsonDocument wrap(final Document document) { - return new BsonDocumentWrapper<>(document, new DocumentCodec()); + return new CollectionHelper<>(new DocumentCodec(), collection.getNamespace()); } } diff --git a/driver-sync/src/test/functional/com/mongodb/client/Fixture.java b/driver-sync/src/test/functional/com/mongodb/client/Fixture.java index 3b0d45dca88..8114d62e41a 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/Fixture.java +++ b/driver-sync/src/test/functional/com/mongodb/client/Fixture.java @@ -16,6 +16,7 @@ package com.mongodb.client; +import com.mongodb.ClusterFixture; import com.mongodb.ConnectionString; import com.mongodb.MongoClientSettings; import com.mongodb.ServerAddress; @@ -24,7 +25,6 @@ import java.util.List; import java.util.concurrent.TimeUnit; -import static com.mongodb.ClusterFixture.getConnectionString; import static com.mongodb.ClusterFixture.getMultiMongosConnectionString; import static com.mongodb.ClusterFixture.getServerApi; import static com.mongodb.internal.connection.ClusterDescriptionHelper.getPrimaries; @@ -34,7 +34,6 @@ * Helper class for the acceptance tests. */ public final class Fixture { - private static final String DEFAULT_DATABASE_NAME = "JavaDriverTest"; private static final long MIN_HEARTBEAT_FREQUENCY_MS = 50L; private static MongoClient mongoClient; @@ -44,10 +43,23 @@ private Fixture() { } public static synchronized MongoClient getMongoClient() { - if (mongoClient == null) { - mongoClient = MongoClients.create(getMongoClientSettings()); - Runtime.getRuntime().addShutdownHook(new ShutdownHook()); + if (mongoClient != null) { + return mongoClient; } + MongoClientSettings mongoClientSettings = getMongoClientSettings(); + mongoClient = MongoClients.create(mongoClientSettings); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + synchronized (Fixture.class) { + if (mongoClient == null) { + return; + } + if (defaultDatabase != null) { + defaultDatabase.drop(); + } + mongoClient.close(); + mongoClient = null; + } + })); return mongoClient; } @@ -59,34 +71,15 @@ public static synchronized MongoDatabase getDefaultDatabase() { } public static String getDefaultDatabaseName() { - return DEFAULT_DATABASE_NAME; - } - - static class ShutdownHook extends Thread { - @Override - public void run() { - synchronized (Fixture.class) { - if (mongoClient != null) { - if (defaultDatabase != null) { - defaultDatabase.drop(); - } - mongoClient.close(); - mongoClient = null; - } - } - } + return ClusterFixture.getDefaultDatabaseName(); } public static MongoClientSettings getMongoClientSettings() { return getMongoClientSettingsBuilder().build(); } - public static MongoClientSettings getMultiMongosMongoClientSettings() { - return getMultiMongosMongoClientSettingsBuilder().build(); - } - public static MongoClientSettings.Builder getMongoClientSettingsBuilder() { - return getMongoClientSettings(getConnectionString()); + return getMongoClientSettings(ClusterFixture.getConnectionString()); } public static MongoClientSettings.Builder getMultiMongosMongoClientSettingsBuilder() { diff --git a/driver-sync/src/test/functional/com/mongodb/client/model/mql/AggregatesSearchTest.java b/driver-sync/src/test/functional/com/mongodb/client/model/mql/AggregatesSearchTest.java new file mode 100644 index 00000000000..65766d3b628 --- /dev/null +++ b/driver-sync/src/test/functional/com/mongodb/client/model/mql/AggregatesSearchTest.java @@ -0,0 +1,249 @@ +/* + * Copyright 2008-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.mongodb.client.model.mql; + +import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.SearchIndexModel; +import com.mongodb.client.model.search.SearchOperator; +import com.mongodb.internal.connection.ServerHelper; +import com.mongodb.lang.Nullable; +import org.bson.BsonArray; +import org.bson.BsonDocument; +import org.bson.Document; +import org.bson.codecs.DecoderContext; +import org.bson.conversions.Bson; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import static com.mongodb.ClusterFixture.isAtlasSearchTest; +import static com.mongodb.ClusterFixture.serverVersionAtLeast; +import static com.mongodb.MongoClientSettings.getDefaultCodecRegistry; +import static com.mongodb.client.Fixture.getMongoClient; +import static com.mongodb.client.Fixture.getPrimary; +import static com.mongodb.client.model.Aggregates.search; +import static com.mongodb.client.model.Aggregates.sort; +import static com.mongodb.client.model.Sorts.ascending; +import static com.mongodb.client.model.search.SearchOptions.searchOptions; +import static com.mongodb.client.model.search.SearchPath.fieldPath; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class AggregatesSearchTest { + public static final String ATLAS_SEARCH_DATABASE = "javaVectorSearchTest"; + private static MongoClient client; + private static MongoDatabase database; + private static MongoCollection collection; + private static String searchIndexName; + + @BeforeAll + public static void beforeAll() { + assumeTrue(isAtlasSearchTest()); + assumeTrue(serverVersionAtLeast(8, 0)); + + client = getMongoClient(); + database = client.getDatabase(ATLAS_SEARCH_DATABASE); + String collectionName = AggregatesSearchTest.class.getName(); + collection = database.getCollection(collectionName); + collection.drop(); + + // We insert documents first. The ensuing indexing guarantees that all + // data present at the time indexing commences will be indexed before + // the index enters the READY state. + insertDocuments("[\n" + + " { _id: 1 },\n" + + " { _id: 2, title: null },\n" + + " { _id: 3, title: 'test' },\n" + + " { _id: 4, title: ['test', 'xyz'] },\n" + + " { _id: 5, title: 'not test' },\n" + + " { _id: 6, description: 'desc 1' },\n" + + " { _id: 7, description: 'desc 8' },\n" + + " { _id: 8, summary: 'summary 1' },\n" + + " { _id: 9, summary: 'summary 2' },\n" + + "]"); + + searchIndexName = "not_default"; + // Index creation can take disproportionately long, so we create it once + // for all tests. + // We set dynamic to true to index unspecified fields. Different kinds + // of fields are needed for different tests. + collection.createSearchIndexes(Arrays.asList(new SearchIndexModel(searchIndexName, Document.parse( + "{\n" + + " \"mappings\": {\n" + + " \"dynamic\": true,\n" + + " \"fields\": {\n" + + " \"title\": {\n" + + " \"type\": \"token\"\n" + + " },\n" + + " \"description\": {\n" + + " \"analyzer\": \"lucene.keyword\"," + + " \"type\": \"string\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}")))); + waitForIndex(collection, searchIndexName); + } + + @AfterAll + public static void afterAll() { + if (collection != null) { + collection.drop(); + } + try { + ServerHelper.checkPool(getPrimary()); + } catch (InterruptedException e) { + // ignore + } + } + + @Test + public void testExists() { + List pipeline = Arrays.asList( + search(SearchOperator.exists(fieldPath("title")), + searchOptions().index(searchIndexName))); + assertResults(pipeline, "[\n" + + " { _id: 2, title: null },\n" + + " { _id: 3, title: 'test' },\n" + + " { _id: 4, title: ['test', 'xyz'] },\n" + + " { _id: 5, title: 'not test' },\n" + + "]"); + } + + @Test + public void testEquals() { + List pipeline1 = Arrays.asList( + search(SearchOperator.equals(fieldPath("title"), "test"), + searchOptions().index(searchIndexName))); + assertResults(pipeline1, "[\n" + + " { _id: 3, title: 'test' }\n" + + " { _id: 4, title: ['test', 'xyz'] }\n" + + "]"); + + // equals null does not match non-existent fields + List pipeline2 = Arrays.asList( + search(SearchOperator.equalsNull(fieldPath("title")), + searchOptions().index(searchIndexName))); + assertResults(pipeline2, "[\n" + + " { _id: 2, title: null }\n" + + "]"); + } + + @Test + public void testMoreLikeThis() { + List pipeline = Arrays.asList( + search(SearchOperator.moreLikeThis(Document.parse("{ summary: 'summary' }").toBsonDocument()), + searchOptions().index(searchIndexName))); + assertResults(pipeline, "[\n" + + " { _id: 8, summary: 'summary 1' },\n" + + " { _id: 9, summary: 'summary 2' },\n" + + "]"); + } + + @Test + public void testRegex() { + List pipeline = Arrays.asList( + search(SearchOperator.regex(fieldPath("description"), "des[c]+ <1-4>"), + searchOptions().index(searchIndexName))); + assertResults(pipeline, "[\n" + + " { _id: 6, description: 'desc 1' },\n" + + "]"); + } + + @Test + public void testWildcard() { + List pipeline = Arrays.asList( + search(SearchOperator.wildcard("desc*", fieldPath("description")), + searchOptions().index(searchIndexName))); + assertResults(pipeline, "[\n" + + " { _id: 6, description: 'desc 1' },\n" + + " { _id: 7, description: 'desc 8' },\n" + + "]"); + } + + private static void insertDocuments(final String s) { + List documents = BsonArray.parse(s).stream() + .map(v -> new Document(v.asDocument())) + .collect(Collectors.toList()); + collection.insertMany(documents); + } + + private static void assertResults(final List pipeline, final String expectedResultsAsString) { + ArrayList pipeline2 = new ArrayList<>(pipeline); + pipeline2.add(sort(ascending("_id"))); + + List expectedResults = parseToList(expectedResultsAsString); + List actualResults = aggregate(pipeline2); + assertEquals(expectedResults, actualResults); + } + + private static List aggregate(final List stages) { + AggregateIterable result = collection.aggregate(stages); + List results = new ArrayList<>(); + result.forEach(r -> results.add(r.toBsonDocument())); + return results; + } + + public static List parseToList(final String s) { + return BsonArray.parse(s).stream().map(v -> toBsonDocument(v.asDocument())).collect(Collectors.toList()); + } + + public static BsonDocument toBsonDocument(final BsonDocument bsonDocument) { + return getDefaultCodecRegistry().get(BsonDocument.class).decode(bsonDocument.asBsonReader(), DecoderContext.builder().build()); + } + + @Nullable + private static Document indexRecord( + final MongoCollection collection, final String indexName) { + return StreamSupport.stream(collection.listSearchIndexes().spliterator(), false) + .filter(index -> indexName.equals(index.getString("name"))) + .findAny().orElse(null); + } + + static void waitForIndex(final MongoCollection collection, final String indexName) { + long startTime = System.nanoTime(); + long timeoutNanos = TimeUnit.SECONDS.toNanos(20); + while (System.nanoTime() - startTime < timeoutNanos) { + Document indexRecord = indexRecord(collection, indexName); + if (indexRecord != null) { + if ("FAILED".equals(indexRecord.getString("status"))) { + throw new RuntimeException("Search index has failed status."); + } + if (indexRecord.getBoolean("queryable")) { + return; + } + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } + +} From ec4a8e444b6c0170cd3549ee0ca87ca6928bad31 Mon Sep 17 00:00:00 2001 From: Maxim Katcharov Date: Tue, 28 Jan 2025 10:03:09 -0700 Subject: [PATCH 2/5] Automated refactor fieldpath order --- .../client/model/search/SearchOperator.java | 18 ++++++++---------- .../model/search/WildcardSearchOperator.java | 2 +- .../model/search/SearchOperatorTest.java | 4 ++-- .../scala/model/search/SearchOperator.scala | 2 +- .../client/model/mql/AggregatesSearchTest.java | 2 +- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/driver-core/src/main/com/mongodb/client/model/search/SearchOperator.java b/driver-core/src/main/com/mongodb/client/model/search/SearchOperator.java index e457454db0c..27a16e81237 100644 --- a/driver-core/src/main/com/mongodb/client/model/search/SearchOperator.java +++ b/driver-core/src/main/com/mongodb/client/model/search/SearchOperator.java @@ -20,27 +20,24 @@ import com.mongodb.annotations.Sealed; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.geojson.Point; - -import java.util.UUID; - import org.bson.BsonBinary; -import org.bson.BsonNull; import org.bson.BsonDocument; +import org.bson.BsonNull; import org.bson.BsonType; import org.bson.Document; import org.bson.conversions.Bson; +import org.bson.types.ObjectId; import java.time.Duration; import java.time.Instant; import java.util.Iterator; - -import org.bson.types.ObjectId; +import java.util.UUID; import static com.mongodb.assertions.Assertions.isTrueArgument; +import static com.mongodb.assertions.Assertions.notNull; import static com.mongodb.internal.Iterables.concat; import static com.mongodb.internal.client.model.Util.combineToBsonValue; import static java.util.Collections.singleton; -import static com.mongodb.assertions.Assertions.notNull; /** * The core part of the {@link Aggregates#search(SearchOperator, SearchOptions) $search} pipeline stage of an aggregation pipeline. @@ -463,14 +460,15 @@ static PhraseSearchOperator phrase(final Iterable paths, f } /** - * Returns a {@link SearchOperator} that performs a search using a special characters in the search string that can match any character. + * Returns a {@link SearchOperator} that performs a search using a special + * characters in the search string that can match any character. * + * @param path The indexed field to be searched. * @param query The string to search for. - * @param path The indexed field to be searched. * @return The requested {@link SearchOperator}. * @mongodb.atlas.manual atlas-search/wildcard/ wildcard operator */ - static WildcardSearchOperator wildcard(final String query, final SearchPath path) { + static WildcardSearchOperator wildcard(final SearchPath path, final String query) { return wildcard(singleton(notNull("query", query)), singleton(notNull("path", path))); } diff --git a/driver-core/src/main/com/mongodb/client/model/search/WildcardSearchOperator.java b/driver-core/src/main/com/mongodb/client/model/search/WildcardSearchOperator.java index 651d9ffa57c..95d4a5caad5 100644 --- a/driver-core/src/main/com/mongodb/client/model/search/WildcardSearchOperator.java +++ b/driver-core/src/main/com/mongodb/client/model/search/WildcardSearchOperator.java @@ -20,7 +20,7 @@ import com.mongodb.annotations.Sealed; /** - * @see SearchOperator#wildcard(String, SearchPath) + * @see SearchOperator#wildcard(SearchPath, String) * @see SearchOperator#wildcard(Iterable, Iterable) * @since 4.7 */ diff --git a/driver-core/src/test/unit/com/mongodb/client/model/search/SearchOperatorTest.java b/driver-core/src/test/unit/com/mongodb/client/model/search/SearchOperatorTest.java index 06a825c6125..dd36a451f6d 100644 --- a/driver-core/src/test/unit/com/mongodb/client/model/search/SearchOperatorTest.java +++ b/driver-core/src/test/unit/com/mongodb/client/model/search/SearchOperatorTest.java @@ -746,8 +746,8 @@ void wildcard() { .append("path", fieldPath("fieldName").toBsonValue()) ), SearchOperator.wildcard( - "term", - fieldPath("fieldName")) + fieldPath("fieldName"), "term" + ) .toBsonDocument() ), () -> assertEquals( diff --git a/driver-scala/src/main/scala/org/mongodb/scala/model/search/SearchOperator.scala b/driver-scala/src/main/scala/org/mongodb/scala/model/search/SearchOperator.scala index 500724d4d31..71848580dc4 100644 --- a/driver-scala/src/main/scala/org/mongodb/scala/model/search/SearchOperator.scala +++ b/driver-scala/src/main/scala/org/mongodb/scala/model/search/SearchOperator.scala @@ -336,7 +336,7 @@ object SearchOperator { * @return The requested `SearchOperator`. * @see [[https://www.mongodb.com/docs/atlas/atlas-search/wildcard/ wildcard operator]] */ - def wildcard(query: String, path: SearchPath): WildcardSearchOperator = JSearchOperator.wildcard(query, path) + def wildcard(query: String, path: SearchPath): WildcardSearchOperator = JSearchOperator.wildcard(path, query) /** * Returns a `SearchOperator` that enables queries which use special characters in the search string that can match any character. diff --git a/driver-sync/src/test/functional/com/mongodb/client/model/mql/AggregatesSearchTest.java b/driver-sync/src/test/functional/com/mongodb/client/model/mql/AggregatesSearchTest.java index 65766d3b628..739154af60c 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/model/mql/AggregatesSearchTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/model/mql/AggregatesSearchTest.java @@ -177,7 +177,7 @@ public void testRegex() { @Test public void testWildcard() { List pipeline = Arrays.asList( - search(SearchOperator.wildcard("desc*", fieldPath("description")), + search(SearchOperator.wildcard(fieldPath("description"), "desc*"), searchOptions().index(searchIndexName))); assertResults(pipeline, "[\n" + " { _id: 6, description: 'desc 1' },\n" From 49c80c1757e50c71722a52533ae28c6ac49b6a6c Mon Sep 17 00:00:00 2001 From: Maxim Katcharov Date: Fri, 31 Jan 2025 11:39:50 -0700 Subject: [PATCH 3/5] Add phrase and queryString tests --- .../model/mql/AggregatesSearchTest.java | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/driver-sync/src/test/functional/com/mongodb/client/model/mql/AggregatesSearchTest.java b/driver-sync/src/test/functional/com/mongodb/client/model/mql/AggregatesSearchTest.java index 739154af60c..e7fd2666a8f 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/model/mql/AggregatesSearchTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/model/mql/AggregatesSearchTest.java @@ -82,8 +82,8 @@ public static void beforeAll() { + " { _id: 5, title: 'not test' },\n" + " { _id: 6, description: 'desc 1' },\n" + " { _id: 7, description: 'desc 8' },\n" - + " { _id: 8, summary: 'summary 1' },\n" - + " { _id: 9, summary: 'summary 2' },\n" + + " { _id: 8, summary: 'summary 1 one five' },\n" + + " { _id: 9, summary: 'summary 2 one two three four five' },\n" + "]"); searchIndexName = "not_default"; @@ -159,8 +159,8 @@ public void testMoreLikeThis() { search(SearchOperator.moreLikeThis(Document.parse("{ summary: 'summary' }").toBsonDocument()), searchOptions().index(searchIndexName))); assertResults(pipeline, "[\n" - + " { _id: 8, summary: 'summary 1' },\n" - + " { _id: 9, summary: 'summary 2' },\n" + + " { _id: 8, summary: 'summary 1 one five' },\n" + + " { _id: 9, summary: 'summary 2 one two three four five' },\n" + "]"); } @@ -185,6 +185,26 @@ public void testWildcard() { + "]"); } + @Test + public void testPhrase() { + List pipeline = Arrays.asList( + search(SearchOperator.phrase(fieldPath("summary"), "one five").slop(2), + searchOptions().index(searchIndexName))); + assertResults(pipeline, "[\n" + + " { _id: 8, summary: 'summary 1 one five' },\n" + + "]"); + } + + @Test + public void testQueryString() { + List pipeline = Arrays.asList( + search(SearchOperator.queryString(fieldPath("summary"), "summary: one AND summary: three"), + searchOptions().index(searchIndexName))); + assertResults(pipeline, "[\n" + + " { _id: 9, summary: 'summary 2 one two three four five' },\n" + + "]"); + } + private static void insertDocuments(final String s) { List documents = BsonArray.parse(s).stream() .map(v -> new Document(v.asDocument())) From 24d7d1edda71b3e1b3b908eed6266e9f943d8858 Mon Sep 17 00:00:00 2001 From: Maxim Katcharov Date: Tue, 4 Feb 2025 11:11:59 -0700 Subject: [PATCH 4/5] Clean up evg --- .evergreen/.evg.yml | 137 ++++++------------ .../AggregatesSearchIntegrationTest.java | 6 +- .../AggregatesSearchFunctionalTest.java} | 13 +- 3 files changed, 53 insertions(+), 103 deletions(-) rename driver-sync/src/test/functional/com/mongodb/client/model/{mql/AggregatesSearchTest.java => search/AggregatesSearchFunctionalTest.java} (96%) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index 4ec4fdcf8c8..7f3ca3d2c53 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -509,37 +509,6 @@ functions: mongo --nodb setup.js aws_e2e_ecs.js cd - - run-atlas-connectivity-test-func: - - command: shell.exec - type: test - params: - silent: true - working_dir: "src" - script: | - # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does) - # The connection strings are pipe-delimited - MONGODB_URIS="${atlas_free_tier_uri}|${atlas_replica_set_uri}|${atlas_sharded_uri}|${atlas_tls_v11_uri}|${atlas_tls_v12_uri}|${atlas_free_tier_uri_srv}|${atlas_replica_set_uri_srv}|${atlas_sharded_uri_srv}|${atlas_tls_v11_uri_srv}|${atlas_tls_v12_uri_srv}|${atlas_serverless_uri}|${atlas_serverless_uri_srv}" \ - JAVA_VERSION="8" \ - .evergreen/run-connectivity-tests.sh - - "run atlas data lake test": - - command: shell.exec - type: test - params: - working_dir: "src" - script: | - ${PREPARE_SHELL} - JAVA_VERSION=${JAVA_VERSION} .evergreen/run-atlas-data-lake-test.sh - - "run atlas search test": - - command: shell.exec - type: test - params: - working_dir: "src" - script: | - ${PREPARE_SHELL} - MONGODB_URI="${atlas_search_uri}" .evergreen/run-atlas-search-tests.sh - "run-ocsp-test": - command: shell.exec type: test @@ -1462,20 +1431,42 @@ tasks: OCSP_MUST_STAPLE: "false" OCSP_TLS_SHOULD_SUCCEED: "0" - - name: "atlas-data-lake-test" + - name: "atlas-data-lake-task" commands: - func: "bootstrap mongohoused" - - func: "run atlas data lake test" + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + JAVA_VERSION=${JAVA_VERSION} .evergreen/run-atlas-data-lake-test.sh - - name: "atlas-search-test" + - name: "atlas-search-task" commands: - - func: "run atlas search test" + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + MONGODB_URI="${atlas_search_uri}" .evergreen/run-atlas-search-tests.sh - - name: "atlas-test-task" + - name: "atlas-connectivity-task" commands: - - func: run-atlas-connectivity-test-func + - command: shell.exec + type: test + params: + silent: true + working_dir: "src" + script: | + # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does) + # The connection strings are pipe-delimited + MONGODB_URIS="${atlas_free_tier_uri}|${atlas_replica_set_uri}|${atlas_sharded_uri}|${atlas_tls_v11_uri}|${atlas_tls_v12_uri}|${atlas_free_tier_uri_srv}|${atlas_replica_set_uri_srv}|${atlas_sharded_uri_srv}|${atlas_tls_v11_uri_srv}|${atlas_tls_v12_uri_srv}|${atlas_serverless_uri}|${atlas_serverless_uri_srv}" \ + JAVA_VERSION="8" \ + .evergreen/run-connectivity-tests.sh - - name: "test-atlas-search-index-helpers" + - name: "atlas-search-index-management-task" commands: - command: subprocess.exec params: @@ -1536,7 +1527,7 @@ tasks: - func: "run perf tests" - func: "send dashboard data" - - name: "test-aws-lambda-deployed" + - name: "aws-lambda-deployed-task" commands: - command: ec2.assume_role params: @@ -1917,11 +1908,12 @@ axes: batchtime: 10080 # 7 days task_groups: - - name: test-atlas-search-task-group + - name: "atlas-deployed-task-group" setup_group: - func: fetch source - func: prepare resources - command: subprocess.exec + type: setup params: working_dir: src binary: bash @@ -1933,6 +1925,7 @@ task_groups: file: src/atlas-expansion.yml teardown_group: - command: subprocess.exec + type: setup params: working_dir: src binary: bash @@ -1942,7 +1935,8 @@ task_groups: setup_group_can_fail_task: true setup_group_timeout_secs: 1800 tasks: - - test-atlas-search-index-helpers + - "atlas-search-index-management-task" + - "aws-lambda-deployed-task" - name: testgcpkms_task_group setup_group_can_fail_task: true @@ -2025,33 +2019,6 @@ task_groups: tasks: - testazurekms-task - - name: test_atlas_task_group - setup_group: - - func: fetch source - - func: prepare resources - - command: subprocess.exec - params: - working_dir: src - binary: bash - add_expansions_to_env: true - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh - - command: expansions.update - params: - file: src/atlas-expansion.yml - teardown_group: - - command: subprocess.exec - params: - working_dir: src - binary: bash - add_expansions_to_env: true - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 - tasks: - - test-aws-lambda-deployed - - name: testoidc_task_group setup_group: - func: fetch source @@ -2285,12 +2252,6 @@ buildvariants: tasks: - name: "perf" -- name: rhel8-test-atlas - display_name: Atlas Cluster Tests - run_on: rhel80-large - tasks: - - test_atlas_task_group - - name: plain-auth-test display_name: "PLAIN (LDAP) Auth test" run_on: rhel80-small @@ -2351,29 +2312,19 @@ buildvariants: tasks: - name: ".ocsp" -- name: atlas-data-lake-test - display_name: "Atlas Data Lake test" - run_on: ubuntu2004-small - tasks: - - name: "atlas-data-lake-test" - -- name: atlas-test-variant - display_name: "Atlas test" - run_on: rhel80-small - tasks: - - name: atlas-test-task - -- name: atlas-search-test - display_name: "Atlas Search test" +- name: "atlas-search-variant" + display_name: "Atlas Tests" run_on: rhel80-small tasks: - - name: "atlas-search-test" + - name: "atlas-deployed-task-group" + - name: "atlas-search-task" + - name: "atlas-connectivity-task" -- name: atlas-search-variant - display_name: "Atlas Search Tests" - run_on: rhel80-small +- name: atlas-data-lake-test + display_name: "Atlas Data Lake test" + run_on: ubuntu2004-small tasks: - - name: "test-atlas-search-task-group" + - name: "atlas-data-lake-task" - name: "reactive-streams-tck-test" display_name: "Reactive Streams TCK tests" diff --git a/driver-core/src/test/functional/com/mongodb/client/model/search/AggregatesSearchIntegrationTest.java b/driver-core/src/test/functional/com/mongodb/client/model/search/AggregatesSearchIntegrationTest.java index be87bca82f6..f55bd45c7d8 100644 --- a/driver-core/src/test/functional/com/mongodb/client/model/search/AggregatesSearchIntegrationTest.java +++ b/driver-core/src/test/functional/com/mongodb/client/model/search/AggregatesSearchIntegrationTest.java @@ -121,9 +121,6 @@ import static org.junit.jupiter.params.provider.Arguments.arguments; /** - * Use this class when needing to test against MFLIX specifically. Otherwise, - * see AggregatesSearchTest. - *

* These tests require the sample data * and the following Atlas Search indices: *

@@ -231,6 +228,9 @@ * * *
+ *

+ * Use this class when needing to test against MFLIX specifically. Otherwise, + * see AggregatesSearchTest. */ final class AggregatesSearchIntegrationTest { private static final MongoNamespace MFLIX_MOVIES_NS = new MongoNamespace("sample_mflix", "movies"); diff --git a/driver-sync/src/test/functional/com/mongodb/client/model/mql/AggregatesSearchTest.java b/driver-sync/src/test/functional/com/mongodb/client/model/search/AggregatesSearchFunctionalTest.java similarity index 96% rename from driver-sync/src/test/functional/com/mongodb/client/model/mql/AggregatesSearchTest.java rename to driver-sync/src/test/functional/com/mongodb/client/model/search/AggregatesSearchFunctionalTest.java index e7fd2666a8f..02a71bc7219 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/model/mql/AggregatesSearchTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/model/search/AggregatesSearchFunctionalTest.java @@ -14,14 +14,13 @@ * limitations under the License. */ -package com.mongodb.client.model.mql; +package com.mongodb.client.model.search; import com.mongodb.client.AggregateIterable; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.SearchIndexModel; -import com.mongodb.client.model.search.SearchOperator; import com.mongodb.internal.connection.ServerHelper; import com.mongodb.lang.Nullable; import org.bson.BsonArray; @@ -53,7 +52,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assumptions.assumeTrue; -public class AggregatesSearchTest { +public class AggregatesSearchFunctionalTest { public static final String ATLAS_SEARCH_DATABASE = "javaVectorSearchTest"; private static MongoClient client; private static MongoDatabase database; @@ -67,7 +66,7 @@ public static void beforeAll() { client = getMongoClient(); database = client.getDatabase(ATLAS_SEARCH_DATABASE); - String collectionName = AggregatesSearchTest.class.getName(); + String collectionName = AggregatesSearchFunctionalTest.class.getName(); collection = database.getCollection(collectionName); collection.drop(); @@ -237,7 +236,7 @@ public static BsonDocument toBsonDocument(final BsonDocument bsonDocument) { } @Nullable - private static Document indexRecord( + private static Document fetchIndexRecord( final MongoCollection collection, final String indexName) { return StreamSupport.stream(collection.listSearchIndexes().spliterator(), false) .filter(index -> indexName.equals(index.getString("name"))) @@ -246,9 +245,9 @@ private static Document indexRecord( static void waitForIndex(final MongoCollection collection, final String indexName) { long startTime = System.nanoTime(); - long timeoutNanos = TimeUnit.SECONDS.toNanos(20); + long timeoutNanos = TimeUnit.SECONDS.toNanos(60); while (System.nanoTime() - startTime < timeoutNanos) { - Document indexRecord = indexRecord(collection, indexName); + Document indexRecord = fetchIndexRecord(collection, indexName); if (indexRecord != null) { if ("FAILED".equals(indexRecord.getString("status"))) { throw new RuntimeException("Search index has failed status."); From facee04602d8f658098cdd42595d4f662ec967aa Mon Sep 17 00:00:00 2001 From: Maxim Katcharov Date: Wed, 5 Feb 2025 10:45:31 -0700 Subject: [PATCH 5/5] Update waitForIndex --- .../AggregatesSearchFunctionalTest.java | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/driver-sync/src/test/functional/com/mongodb/client/model/search/AggregatesSearchFunctionalTest.java b/driver-sync/src/test/functional/com/mongodb/client/model/search/AggregatesSearchFunctionalTest.java index 02a71bc7219..1513d5495bc 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/model/search/AggregatesSearchFunctionalTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/model/search/AggregatesSearchFunctionalTest.java @@ -22,7 +22,6 @@ import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.SearchIndexModel; import com.mongodb.internal.connection.ServerHelper; -import com.mongodb.lang.Nullable; import org.bson.BsonArray; import org.bson.BsonDocument; import org.bson.Document; @@ -235,34 +234,29 @@ public static BsonDocument toBsonDocument(final BsonDocument bsonDocument) { return getDefaultCodecRegistry().get(BsonDocument.class).decode(bsonDocument.asBsonReader(), DecoderContext.builder().build()); } - @Nullable - private static Document fetchIndexRecord( - final MongoCollection collection, final String indexName) { - return StreamSupport.stream(collection.listSearchIndexes().spliterator(), false) - .filter(index -> indexName.equals(index.getString("name"))) - .findAny().orElse(null); - } - - static void waitForIndex(final MongoCollection collection, final String indexName) { + public static boolean waitForIndex(final MongoCollection collection, final String indexName) { long startTime = System.nanoTime(); long timeoutNanos = TimeUnit.SECONDS.toNanos(60); while (System.nanoTime() - startTime < timeoutNanos) { - Document indexRecord = fetchIndexRecord(collection, indexName); + Document indexRecord = StreamSupport.stream(collection.listSearchIndexes().spliterator(), false) + .filter(index -> indexName.equals(index.getString("name"))) + .findAny().orElse(null); if (indexRecord != null) { if ("FAILED".equals(indexRecord.getString("status"))) { throw new RuntimeException("Search index has failed status."); } if (indexRecord.getBoolean("queryable")) { - return; + return true; } } try { - Thread.sleep(100); + Thread.sleep(100); // busy-wait, avoid in production } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException(e); } } + return false; } }