diff --git a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/IndexingTest.java b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/IndexingTest.java index ce13dfb937a..ea36090853e 100644 --- a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/IndexingTest.java +++ b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/IndexingTest.java @@ -14,7 +14,13 @@ package com.google.firebase.firestore; +import static com.google.firebase.firestore.testutil.IntegrationTestUtil.testCollectionWithDocs; import static com.google.firebase.firestore.testutil.IntegrationTestUtil.testFirestore; +import static com.google.firebase.firestore.testutil.IntegrationTestUtil.waitFor; +import static com.google.firebase.firestore.testutil.TestUtil.assertDoesNotThrow; +import static com.google.firebase.firestore.testutil.TestUtil.expectError; +import static com.google.firebase.firestore.testutil.TestUtil.map; +import static org.junit.Assert.assertEquals; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.android.gms.tasks.Task; @@ -101,4 +107,73 @@ public void testBadIndexDoesNotCrashClient() { + " \"fieldOverrides\": []\n" + "}")); } + + @Test + public void testAutoIndexCreationSetSuccessfully() { + // Use persistent disk cache (default) + FirebaseFirestore db = testFirestore(); + FirebaseFirestoreSettings settings = + new FirebaseFirestoreSettings.Builder(db.getFirestoreSettings()) + .setLocalCacheSettings(PersistentCacheSettings.newBuilder().build()) + .build(); + db.setFirestoreSettings(settings); + + CollectionReference collection = + testCollectionWithDocs( + map( + "a", map("match", true), + "b", map("match", false), + "c", map("match", false))); + QuerySnapshot results = waitFor(collection.whereEqualTo("match", true).get()); + assertEquals(1, results.size()); + + assertDoesNotThrow(() -> db.getPersistentCacheIndexManager().enableIndexAutoCreation()); + + results = waitFor(collection.whereEqualTo("match", true).get()); + assertEquals(1, results.size()); + + assertDoesNotThrow(() -> db.getPersistentCacheIndexManager().disableIndexAutoCreation()); + + results = waitFor(collection.whereEqualTo("match", true).get()); + assertEquals(1, results.size()); + } + + @Test + public void testAutoIndexCreationSetSuccessfullyUsingDefault() { + // Use persistent disk cache (default) + FirebaseFirestore db = testFirestore(); + + CollectionReference collection = + testCollectionWithDocs( + map( + "a", map("match", true), + "b", map("match", false), + "c", map("match", false))); + QuerySnapshot results = waitFor(collection.whereEqualTo("match", true).get()); + assertEquals(1, results.size()); + + assertDoesNotThrow(() -> db.getPersistentCacheIndexManager().enableIndexAutoCreation()); + + results = waitFor(collection.whereEqualTo("match", true).get()); + assertEquals(1, results.size()); + + assertDoesNotThrow(() -> db.getPersistentCacheIndexManager().disableIndexAutoCreation()); + + results = waitFor(collection.whereEqualTo("match", true).get()); + assertEquals(1, results.size()); + } + + @Test + public void testAutoIndexCreationAfterFailsTermination() { + FirebaseFirestore db = testFirestore(); + waitFor(db.terminate()); + + expectError( + () -> db.getPersistentCacheIndexManager().enableIndexAutoCreation(), + "The client has already been terminated"); + + expectError( + () -> db.getPersistentCacheIndexManager().disableIndexAutoCreation(), + "The client has already been terminated"); + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestore.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestore.java index 0020215ef27..83fac435e86 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestore.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestore.java @@ -23,6 +23,7 @@ import androidx.annotation.Keep; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import androidx.annotation.RestrictTo; import androidx.annotation.VisibleForTesting; import com.google.android.gms.tasks.Task; import com.google.android.gms.tasks.TaskCompletionSource; @@ -104,6 +105,8 @@ public interface InstanceRegistry { private volatile FirestoreClient client; private final GrpcMetadataProvider metadataProvider; + @Nullable private PersistentCacheIndexManager persistentCacheIndexManager; + @NonNull private static FirebaseApp getDefaultFirebaseApp() { FirebaseApp app = FirebaseApp.getInstance(); @@ -403,6 +406,26 @@ public Task setIndexConfiguration(@NonNull String json) { return client.configureFieldIndexes(parsedIndexes); } + /** + * Returns the PersistentCache Index Manager used by this {@code FirebaseFirestore} object. + * + * @return The {@code PersistentCacheIndexManager} instance or null if local persistent storage is + * not in use. + */ + // TODO(csi): Remove the `hide` and scope annotations. + /** @hide */ + @RestrictTo(RestrictTo.Scope.LIBRARY) + @Nullable + public synchronized PersistentCacheIndexManager getPersistentCacheIndexManager() { + ensureClientConfigured(); + if (persistentCacheIndexManager == null + && (settings.isPersistenceEnabled() + || settings.getCacheSettings() instanceof PersistentCacheSettings)) { + persistentCacheIndexManager = new PersistentCacheIndexManager(client); + } + return persistentCacheIndexManager; + } + /** * Gets a {@code CollectionReference} instance that refers to the collection at the specified path * within the database. diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/PersistentCacheIndexManager.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/PersistentCacheIndexManager.java new file mode 100644 index 00000000000..d9251ba45a3 --- /dev/null +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/PersistentCacheIndexManager.java @@ -0,0 +1,54 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.firebase.firestore; + +import androidx.annotation.NonNull; +import androidx.annotation.RestrictTo; +import com.google.firebase.firestore.core.FirestoreClient; + +/** + * A {@code PersistentCacheIndexManager} which you can config persistent cache indexes used for + * local query execution. + * + *

To use, call {@link FirebaseFirestore#getPersistentCacheIndexManager()} to get an instance. + */ +// TODO(csi): Remove the `hide` and scope annotations. +/** @hide */ +@RestrictTo(RestrictTo.Scope.LIBRARY) +public final class PersistentCacheIndexManager { + @NonNull private FirestoreClient client; + + PersistentCacheIndexManager(FirestoreClient client) { + this.client = client; + } + + /** + * Enables SDK to create persistent cache indexes automatically for local query execution when SDK + * believes cache indexes can help improves performance. + * + *

This feature is disabled by default. + */ + public void enableIndexAutoCreation() { + client.setIndexAutoCreationEnabled(true); + } + + /** + * Stops creating persistent cache indexes automatically for local query execution. The indexes + * which have been created by calling {@link #enableIndexAutoCreation()} still take effect. + */ + public void disableIndexAutoCreation() { + client.setIndexAutoCreationEnabled(false); + } +} diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java index 546300c5acc..9367712d991 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java @@ -353,6 +353,11 @@ public Task configureFieldIndexes(List fieldIndices) { return asyncQueue.enqueue(() -> localStore.configureFieldIndexes(fieldIndices)); } + public void setIndexAutoCreationEnabled(boolean enabled) { + verifyNotTerminated(); + asyncQueue.enqueueAndForget(() -> localStore.setIndexAutoCreationEnabled(enabled)); + } + public void removeSnapshotsInSyncListener(EventListener listener) { // Checks for shutdown but does not raise error, allowing remove after shutdown to be a no-op. if (isTerminated()) { diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexManager.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexManager.java index 3e5fa08b08e..8ba45cd6efd 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexManager.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexManager.java @@ -78,6 +78,9 @@ enum IndexType { /** Removes the given field index and deletes all index values. */ void deleteFieldIndex(FieldIndex index); + /** Creates a full matched field index which serves the given target. */ + void createTargetIndexes(Target target); + /** * Returns a list of field indexes that correspond to the specified collection group. * diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalDocumentsView.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalDocumentsView.java index 274285b975c..048d2fc06aa 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalDocumentsView.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalDocumentsView.java @@ -258,19 +258,32 @@ void recalculateAndSaveOverlays(Set documentKeys) { * * @param query The query to match documents against. * @param offset Read time and key to start scanning by (exclusive). + * @param context A optional tracker to keep a record of important details during database local + * query execution. */ ImmutableSortedMap getDocumentsMatchingQuery( - Query query, IndexOffset offset) { + Query query, IndexOffset offset, @Nullable QueryContext context) { ResourcePath path = query.getPath(); if (query.isDocumentQuery()) { return getDocumentsMatchingDocumentQuery(path); } else if (query.isCollectionGroupQuery()) { - return getDocumentsMatchingCollectionGroupQuery(query, offset); + return getDocumentsMatchingCollectionGroupQuery(query, offset, context); } else { - return getDocumentsMatchingCollectionQuery(query, offset); + return getDocumentsMatchingCollectionQuery(query, offset, context); } } + /** + * Performs a query against the local view of all documents. + * + * @param query The query to match documents against. + * @param offset Read time and key to start scanning by (exclusive). + */ + ImmutableSortedMap getDocumentsMatchingQuery( + Query query, IndexOffset offset) { + return getDocumentsMatchingQuery(query, offset, /*context*/ null); + } + /** Performs a simple document lookup for the given path. */ private ImmutableSortedMap getDocumentsMatchingDocumentQuery( ResourcePath path) { @@ -284,7 +297,7 @@ private ImmutableSortedMap getDocumentsMatchingDocumentQu } private ImmutableSortedMap getDocumentsMatchingCollectionGroupQuery( - Query query, IndexOffset offset) { + Query query, IndexOffset offset, @Nullable QueryContext context) { hardAssert( query.getPath().isEmpty(), "Currently we only support collection group queries at the root."); @@ -297,7 +310,7 @@ private ImmutableSortedMap getDocumentsMatchingCollection for (ResourcePath parent : parents) { Query collectionQuery = query.asCollectionQueryAtPath(parent.append(collectionId)); ImmutableSortedMap collectionResults = - getDocumentsMatchingCollectionQuery(collectionQuery, offset); + getDocumentsMatchingCollectionQuery(collectionQuery, offset, context); for (Map.Entry docEntry : collectionResults) { results = results.insert(docEntry.getKey(), docEntry.getValue()); } @@ -362,11 +375,11 @@ private void populateOverlays(Map overlays, Set getDocumentsMatchingCollectionQuery( - Query query, IndexOffset offset) { + Query query, IndexOffset offset, @Nullable QueryContext context) { Map overlays = documentOverlayCache.getOverlays(query.getPath(), offset.getLargestBatchId()); Map remoteDocuments = - remoteDocumentCache.getDocumentsMatchingQuery(query, offset, overlays.keySet()); + remoteDocumentCache.getDocumentsMatchingQuery(query, offset, overlays.keySet(), context); // As documents might match the query because of their overlay we need to include documents // for all overlays in the initial document set. diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java index 81b7dd13455..c0be00e8111 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java @@ -802,6 +802,10 @@ public void configureFieldIndexes(List newFieldIndexes) { }); } + public void setIndexAutoCreationEnabled(boolean enabled) { + queryEngine.setIndexAutoCreationEnabled(enabled); + } + /** Mutable state for the transaction in allocateQuery. */ private static class AllocateQueryHolder { TargetData cached; diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryIndexManager.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryIndexManager.java index 9755abbf0b3..0cb81deae8b 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryIndexManager.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryIndexManager.java @@ -60,6 +60,9 @@ public void deleteFieldIndex(FieldIndex index) { // Field indices are not supported with memory persistence. } + @Override + public void createTargetIndexes(Target target) {} + @Override @Nullable public List getDocumentsMatchingTarget(Target target) { diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java index 3452e482dc2..3b4fa1048b2 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java @@ -18,6 +18,7 @@ import static com.google.firebase.firestore.util.Assert.hardAssert; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import com.google.firebase.database.collection.ImmutableSortedMap; import com.google.firebase.firestore.core.Query; import com.google.firebase.firestore.model.Document; @@ -97,7 +98,10 @@ public Map getAll( @Override public Map getDocumentsMatchingQuery( - Query query, IndexOffset offset, @Nonnull Set mutatedKeys) { + Query query, + IndexOffset offset, + @Nonnull Set mutatedKeys, + @Nullable QueryContext context) { Map result = new HashMap<>(); // Documents are ordered by key, so we can use a prefix scan to narrow down the documents @@ -135,6 +139,12 @@ public Map getDocumentsMatchingQuery( return result; } + @Override + public Map getDocumentsMatchingQuery( + Query query, IndexOffset offset, @Nonnull Set mutatedKeys) { + return getDocumentsMatchingQuery(query, offset, mutatedKeys, /*context*/ null); + } + Iterable getDocuments() { return new DocumentIterable(); } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/QueryContext.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/QueryContext.java new file mode 100644 index 00000000000..ed25b1fa07a --- /dev/null +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/QueryContext.java @@ -0,0 +1,29 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.firebase.firestore.local; + +/** A tracker to keep a record of important details during database local query execution. */ +public class QueryContext { + /** Counts the number of documents passed through during local query execution. */ + private int documentReadCount = 0; + + public int getDocumentReadCount() { + return documentReadCount; + } + + public void incrementDocumentReadCount() { + documentReadCount++; + } +} diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/QueryEngine.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/QueryEngine.java index 114549a62e5..77546678f33 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/QueryEngine.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/QueryEngine.java @@ -16,6 +16,7 @@ import static com.google.firebase.firestore.util.Assert.hardAssert; +import androidx.annotation.VisibleForTesting; import com.google.firebase.database.collection.ImmutableSortedMap; import com.google.firebase.database.collection.ImmutableSortedSet; import com.google.firebase.firestore.core.Query; @@ -61,16 +62,36 @@ public class QueryEngine { private static final String LOG_TAG = "QueryEngine"; + private static final int DEFAULT_INDEX_AUTO_CREATION_MIN_COLLECTION_SIZE = 100; + + /** + * This cost represents the evaluation result of (([index, docKey] + [docKey, docContent]) per + * document in the result set) / ([docKey, docContent] per documents in full collection scan) + * coming from experiment https://github.com/firebase/firebase-android-sdk/pull/5064. + */ + private static final double DEFAULT_RELATIVE_INDEX_READ_COST_PER_DOCUMENT = 2; + private LocalDocumentsView localDocumentsView; private IndexManager indexManager; private boolean initialized; + private boolean indexAutoCreationEnabled = false; + + /** SDK only decides whether it should create index when collection size is larger than this. */ + private int indexAutoCreationMinCollectionSize = DEFAULT_INDEX_AUTO_CREATION_MIN_COLLECTION_SIZE; + + private double relativeIndexReadCostPerDocument = DEFAULT_RELATIVE_INDEX_READ_COST_PER_DOCUMENT; + public void initialize(LocalDocumentsView localDocumentsView, IndexManager indexManager) { this.localDocumentsView = localDocumentsView; this.indexManager = indexManager; this.initialized = true; } + public void setIndexAutoCreationEnabled(boolean enabled) { + this.indexAutoCreationEnabled = enabled; + } + public ImmutableSortedMap getDocumentsMatchingQuery( Query query, SnapshotVersion lastLimboFreeSnapshotVersion, @@ -87,7 +108,51 @@ public ImmutableSortedMap getDocumentsMatchingQuery( return result; } - return executeFullCollectionScan(query); + QueryContext context = new QueryContext(); + result = executeFullCollectionScan(query, context); + if (result != null && indexAutoCreationEnabled) { + createCacheIndexes(query, context, result.size()); + } + return result; + } + + /** + * Decides whether SDK should create a full matched field index for this query based on query + * context and query result size. + */ + // TODO(csi): Auto experiment data. + private void createCacheIndexes(Query query, QueryContext context, int resultSize) { + if (context.getDocumentReadCount() < indexAutoCreationMinCollectionSize) { + Logger.debug( + LOG_TAG, + "SDK will not create cache indexes for query: %s, since it only creates cache indexes " + + "for collection contains more than or equal to %s documents.", + query.toString(), + indexAutoCreationMinCollectionSize); + return; + } + + Logger.debug( + LOG_TAG, + "Query: %s, scans %s local documents and returns %s documents as results.", + query.toString(), + context.getDocumentReadCount(), + resultSize); + + if (context.getDocumentReadCount() > relativeIndexReadCostPerDocument * resultSize) { + indexManager.createTargetIndexes(query.toTarget()); + Logger.debug( + LOG_TAG, + "The SDK decides to create cache indexes for query: %s, as using cache indexes " + + "may help improve performance.", + query.toString()); + } else { + Logger.debug( + LOG_TAG, + "The SDK decides not to create cache indexes for this query: %s, as using cache " + + "indexes may not help improve performance.", + query.toString()); + } } /** @@ -241,11 +306,12 @@ private boolean needsRefill( || documentAtLimitEdge.getVersion().compareTo(limboFreeSnapshotVersion) > 0; } - private ImmutableSortedMap executeFullCollectionScan(Query query) { + private ImmutableSortedMap executeFullCollectionScan( + Query query, QueryContext context) { if (Logger.isDebugEnabled()) { Logger.debug(LOG_TAG, "Using full collection scan to execute query: %s", query.toString()); } - return localDocumentsView.getDocumentsMatchingQuery(query, IndexOffset.NONE); + return localDocumentsView.getDocumentsMatchingQuery(query, IndexOffset.NONE, context); } /** @@ -262,4 +328,14 @@ private ImmutableSortedMap appendRemainingResults( } return remainingResults; } + + @VisibleForTesting + void setIndexAutoCreationMinCollectionSize(int newMin) { + indexAutoCreationMinCollectionSize = newMin; + } + + @VisibleForTesting + void setRelativeIndexReadCostPerDocument(double newCost) { + relativeIndexReadCostPerDocument = newCost; + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/RemoteDocumentCache.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/RemoteDocumentCache.java index c4763718593..8ff90864342 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/RemoteDocumentCache.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/RemoteDocumentCache.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.Set; import javax.annotation.Nonnull; +import javax.annotation.Nullable; /** * Represents cached documents received from the remote backend. @@ -89,4 +90,21 @@ interface RemoteDocumentCache { */ Map getDocumentsMatchingQuery( Query query, IndexOffset offset, @Nonnull Set mutatedKeys); + + /** + * Returns the documents that match the given query. + * + * @param query The query to match against remote documents. + * @param offset The read time and document key to start scanning at (exclusive). + * @param mutatedKeys The keys of documents who have mutations attached, they should be read + * regardless whether they match the given query. + * @param context A optional tracker to keep a record of important details during database local + * query execution. + * @return A newly created map with the set of documents in the collection. + */ + Map getDocumentsMatchingQuery( + Query query, + IndexOffset offset, + @Nonnull Set mutatedKeys, + @Nullable QueryContext context); } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteIndexManager.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteIndexManager.java index 8f0d79f4ecd..5c3e4a844a4 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteIndexManager.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteIndexManager.java @@ -233,6 +233,19 @@ public void deleteFieldIndex(FieldIndex index) { } } + @Override + public void createTargetIndexes(Target target) { + hardAssert(started, "IndexManager not started"); + + for (Target subTarget : getSubTargets(target)) { + IndexType type = getIndexType(subTarget); + if (type == IndexType.NONE || type == IndexType.PARTIAL) { + TargetIndexMatcher targetIndexMatcher = new TargetIndexMatcher(subTarget); + addFieldIndex(targetIndexMatcher.buildTargetIndex()); + } + } + } + @Override public @Nullable String getNextCollectionGroupToUpdate() { hardAssert(started, "IndexManager not started"); diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteRemoteDocumentCache.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteRemoteDocumentCache.java index cf687bd3113..b26f9601a81 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteRemoteDocumentCache.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteRemoteDocumentCache.java @@ -182,7 +182,8 @@ private Map getAll( List collections, IndexOffset offset, int count, - @Nullable Function filter) { + @Nullable Function filter, + @Nullable QueryContext context) { Timestamp readTime = offset.getReadTime().getTimestamp(); DocumentKey documentKey = offset.getDocumentKey(); @@ -218,11 +219,25 @@ private Map getAll( Map results = new HashMap<>(); db.query(sql.toString()) .binding(bindVars) - .forEach(row -> processRowInBackground(backgroundQueue, results, row, filter)); + .forEach( + row -> { + processRowInBackground(backgroundQueue, results, row, filter); + if (context != null) { + context.incrementDocumentReadCount(); + } + }); backgroundQueue.drain(); return results; } + private Map getAll( + List collections, + IndexOffset offset, + int count, + @Nullable Function filter) { + return getAll(collections, offset, count, filter, /*context*/ null); + } + private void processRowInBackground( BackgroundQueue backgroundQueue, Map results, @@ -250,11 +265,21 @@ private void processRowInBackground( @Override public Map getDocumentsMatchingQuery( Query query, IndexOffset offset, @Nonnull Set mutatedKeys) { + return getDocumentsMatchingQuery(query, offset, mutatedKeys, /*context*/ null); + } + + @Override + public Map getDocumentsMatchingQuery( + Query query, + IndexOffset offset, + @Nonnull Set mutatedKeys, + @Nullable QueryContext context) { return getAll( Collections.singletonList(query.getPath()), offset, Integer.MAX_VALUE, - (MutableDocument doc) -> query.matches(doc) || mutatedKeys.contains(doc.getKey())); + (MutableDocument doc) -> query.matches(doc) || mutatedKeys.contains(doc.getKey()), + context); } private MutableDocument decodeMaybeDocument( diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/TargetIndexMatcher.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/TargetIndexMatcher.java index c23313a8da0..c4f93616ba1 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/TargetIndexMatcher.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/TargetIndexMatcher.java @@ -192,6 +192,59 @@ public boolean servedByIndex(FieldIndex index) { return true; } + /** Returns a full matched field index for this target. */ + public FieldIndex buildTargetIndex() { + // We want to make sure only one segment created for one field. For example, in case like + // a == 3 and a > 2, index, a ASCENDING, will only be created once. + Set uniqueFields = new HashSet<>(); + List segments = new ArrayList<>(); + + for (FieldFilter filter : equalityFilters) { + if (filter.getField().isKeyField()) { + continue; + } + boolean isArrayOperator = + filter.getOperator().equals(FieldFilter.Operator.ARRAY_CONTAINS) + || filter.getOperator().equals(FieldFilter.Operator.ARRAY_CONTAINS_ANY); + if (isArrayOperator) { + segments.add( + FieldIndex.Segment.create(filter.getField(), FieldIndex.Segment.Kind.CONTAINS)); + } else { + if (uniqueFields.contains(filter.getField())) { + continue; + } + uniqueFields.add(filter.getField()); + segments.add( + FieldIndex.Segment.create(filter.getField(), FieldIndex.Segment.Kind.ASCENDING)); + } + } + + for (OrderBy orderBy : orderBys) { + // Stop adding more segments if we see a order-by on key. Typically this is the default + // implicit order-by which is covered in the index_entry table as a separate column. + // If it is not the default order-by, the generated index will be missing some segments + // optimized for order-bys, which is probably fine. + if (orderBy.getField().isKeyField()) { + continue; + } + + if (uniqueFields.contains(orderBy.getField())) { + continue; + } + uniqueFields.add(orderBy.getField()); + + segments.add( + FieldIndex.Segment.create( + orderBy.getField(), + orderBy.getDirection() == OrderBy.Direction.ASCENDING + ? FieldIndex.Segment.Kind.ASCENDING + : FieldIndex.Segment.Kind.DESCENDING)); + } + + return FieldIndex.create( + FieldIndex.UNKNOWN_ID, collectionId, segments, FieldIndex.INITIAL_STATE); + } + private boolean hasMatchingEqualityFilter(FieldIndex.Segment segment) { for (FieldFilter filter : equalityFilters) { if (matchesFilter(filter, segment)) { diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/CountingQueryEngine.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/CountingQueryEngine.java index a461edcbbda..61385eff006 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/CountingQueryEngine.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/CountingQueryEngine.java @@ -14,6 +14,7 @@ package com.google.firebase.firestore.local; +import androidx.annotation.NonNull; import androidx.annotation.Nullable; import com.google.firebase.database.collection.ImmutableSortedMap; import com.google.firebase.database.collection.ImmutableSortedSet; @@ -86,6 +87,21 @@ public ImmutableSortedMap getDocumentsMatchingQuery( return queryEngine.getDocumentsMatchingQuery(query, lastLimboFreeSnapshotVersion, remoteKeys); } + @Override + public void setIndexAutoCreationEnabled(boolean enabled) { + queryEngine.setIndexAutoCreationEnabled(enabled); + } + + @Override + public void setIndexAutoCreationMinCollectionSize(int newMin) { + queryEngine.setIndexAutoCreationMinCollectionSize(newMin); + } + + @Override + public void setRelativeIndexReadCostPerDocument(double newCost) { + queryEngine.setRelativeIndexReadCostPerDocument(newCost); + } + /** * Returns the number of documents returned by the RemoteDocumentCache's `getAll()` API (since the * last call to `resetCounts()`) @@ -169,9 +185,18 @@ public Map getAll( @Override public Map getDocumentsMatchingQuery( - Query query, IndexOffset offset, Set mutatedKeys) { + Query query, IndexOffset offset, @NonNull Set mutatedKeys) { + return getDocumentsMatchingQuery(query, offset, mutatedKeys, /*context*/ null); + } + + @Override + public Map getDocumentsMatchingQuery( + Query query, + IndexOffset offset, + @NonNull Set mutatedKeys, + @Nullable QueryContext context) { Map result = - subject.getDocumentsMatchingQuery(query, offset, mutatedKeys); + subject.getDocumentsMatchingQuery(query, offset, mutatedKeys, context); documentsReadByCollection[0] += result.size(); return result; } diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LocalStoreTestCase.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LocalStoreTestCase.java index dfff794aaff..4f8645add10 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LocalStoreTestCase.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LocalStoreTestCase.java @@ -161,6 +161,10 @@ protected void backfillIndexes() { indexBackfiller.backfill(); } + protected void setBackfillerMaxDocumentsToProcess(int newMax) { + indexBackfiller.setMaxDocumentsToProcess(newMax); + } + private void updateViews(int targetId, boolean fromCache) { notifyLocalViewChanges(viewChanges(targetId, fromCache, asList(), asList())); } @@ -213,6 +217,23 @@ protected void executeQuery(Query query) { lastQueryResult = localStore.executeQuery(query, /* usePreviousResults= */ true); } + protected void setIndexAutoCreationEnabled(boolean enabled) { + // Noted: there are two queryEngines here, the first one is extended by CountingQueryEngine, + // which is set by localStore function; The second one a pointer inside CountingQueryEngine, + // which is set by queryEngine function. + // Only the second function takes effect in the tests. Adding first one here for compatibility. + localStore.setIndexAutoCreationEnabled(enabled); + queryEngine.setIndexAutoCreationEnabled(enabled); + } + + protected void setMinCollectionSizeToAutoCreateIndex(int newMin) { + queryEngine.setIndexAutoCreationMinCollectionSize(newMin); + } + + protected void setRelativeIndexReadCostPerDocument(double newCost) { + queryEngine.setRelativeIndexReadCostPerDocument(newCost); + } + private void releaseTarget(int targetId) { localStore.releaseTarget(targetId); } diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteLocalStoreTest.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteLocalStoreTest.java index f522469ff18..2885b8aa412 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteLocalStoreTest.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteLocalStoreTest.java @@ -366,4 +366,247 @@ public void testDeeplyNestedServerTimestamps() { } assertThat(error.get()).isNull(); } + + @Test + public void testCanAutoCreateIndexes() { + Query query = query("coll").filter(filter("matches", "==", true)); + int targetId = allocateQuery(query); + + setIndexAutoCreationEnabled(true); + setMinCollectionSizeToAutoCreateIndex(0); + setRelativeIndexReadCostPerDocument(2); + + applyRemoteEvent(addedRemoteEvent(doc("coll/a", 10, map("matches", true)), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/b", 10, map("matches", false)), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/c", 10, map("matches", false)), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/d", 10, map("matches", false)), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/e", 10, map("matches", true)), targetId)); + + // First time query runs without indexes. + // Based on current heuristic, collection document counts (5) > 2 * resultSize (2). + // Full matched index should be created. + executeQuery(query); + assertRemoteDocumentsRead(/* byKey= */ 0, /* byCollection= */ 2); + assertQueryReturned("coll/a", "coll/e"); + + backfillIndexes(); + + applyRemoteEvent(addedRemoteEvent(doc("coll/f", 20, map("matches", true)), targetId)); + + executeQuery(query); + assertRemoteDocumentsRead(/* byKey= */ 2, /* byCollection= */ 1); + assertQueryReturned("coll/a", "coll/e", "coll/f"); + } + + @Test + public void testDoesNotAutoCreateIndexesForSmallCollections() { + Query query = query("coll").filter(filter("count", ">=", 3)); + int targetId = allocateQuery(query); + + setIndexAutoCreationEnabled(true); + setRelativeIndexReadCostPerDocument(2); + + applyRemoteEvent(addedRemoteEvent(doc("coll/a", 10, map("count", 5)), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/b", 10, map("count", 1)), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/c", 10, map("count", 0)), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/d", 10, map("count", 1)), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/e", 10, map("count", 3)), targetId)); + + // SDK will not create indexes since collection size is too small. + executeQuery(query); + assertRemoteDocumentsRead(/* byKey= */ 0, /* byCollection= */ 2); + assertQueryReturned("coll/a", "coll/e"); + + backfillIndexes(); + + applyRemoteEvent(addedRemoteEvent(doc("coll/f", 20, map("count", 4)), targetId)); + + executeQuery(query); + assertRemoteDocumentsRead(/* byKey= */ 0, /* byCollection= */ 3); + assertQueryReturned("coll/a", "coll/e", "coll/f"); + } + + @Test + public void testDoesNotAutoCreateIndexesWhenIndexLookUpIsExpensive() { + Query query = query("coll").filter(filter("array", "array-contains-any", Arrays.asList(0, 7))); + int targetId = allocateQuery(query); + + setIndexAutoCreationEnabled(true); + setMinCollectionSizeToAutoCreateIndex(0); + setRelativeIndexReadCostPerDocument(5); + + applyRemoteEvent( + addedRemoteEvent(doc("coll/a", 10, map("array", Arrays.asList(2, 7))), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/b", 10, map("array", emptyList())), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/c", 10, map("array", singletonList(3))), targetId)); + applyRemoteEvent( + addedRemoteEvent(doc("coll/d", 10, map("array", Arrays.asList(2, 10, 20))), targetId)); + applyRemoteEvent( + addedRemoteEvent(doc("coll/e", 10, map("array", Arrays.asList(2, 0, 8))), targetId)); + + // First time query runs without indexes. + // Based on current heuristic, collection document counts (5) > 2 * resultSize (2). + // Full matched index should be created. + executeQuery(query); + assertRemoteDocumentsRead(/* byKey= */ 0, /* byCollection= */ 2); + assertQueryReturned("coll/a", "coll/e"); + + backfillIndexes(); + + applyRemoteEvent(addedRemoteEvent(doc("coll/f", 20, map("array", singletonList(0))), targetId)); + + executeQuery(query); + assertRemoteDocumentsRead(/* byKey= */ 0, /* byCollection= */ 3); + assertQueryReturned("coll/a", "coll/e", "coll/f"); + } + + @Test + public void testIndexAutoCreationWorksWhenBackfillerRunsHalfway() { + Query query = query("coll").filter(filter("matches", "==", "foo")); + int targetId = allocateQuery(query); + + setIndexAutoCreationEnabled(true); + setMinCollectionSizeToAutoCreateIndex(0); + setRelativeIndexReadCostPerDocument(2); + + applyRemoteEvent(addedRemoteEvent(doc("coll/a", 10, map("matches", "foo")), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/b", 10, map("matches", "")), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/c", 10, map("matches", "bar")), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/d", 10, map("matches", 7)), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/e", 10, map("matches", "foo")), targetId)); + + // First time query is running without indexes. + // Based on current heuristic, collection document counts (5) > 2 * resultSize (2). + // Full matched index should be created. + executeQuery(query); + // Only document a matches the result + assertRemoteDocumentsRead(/* byKey= */ 0, /* byCollection= */ 2); + assertQueryReturned("coll/a", "coll/e"); + + setBackfillerMaxDocumentsToProcess(2); + backfillIndexes(); + + applyRemoteEvent(addedRemoteEvent(doc("coll/f", 20, map("matches", "foo")), targetId)); + + executeQuery(query); + assertRemoteDocumentsRead(/* byKey= */ 1, /* byCollection= */ 2); + assertQueryReturned("coll/a", "coll/e", "coll/f"); + } + + @Test + public void testIndexCreatedByIndexAutoCreationExistsAfterTurnOffAutoCreation() { + Query query = query("coll").filter(filter("value", "not-in", Collections.singletonList(3))); + int targetId = allocateQuery(query); + + setIndexAutoCreationEnabled(true); + setMinCollectionSizeToAutoCreateIndex(0); + setRelativeIndexReadCostPerDocument(2); + + applyRemoteEvent(addedRemoteEvent(doc("coll/a", 10, map("value", 5)), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/b", 10, map("value", 3)), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/c", 10, map("value", 3)), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/d", 10, map("value", 3)), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/e", 10, map("value", 2)), targetId)); + + // First time query runs without indexes. + // Based on current heuristic, collection document counts (5) > 2 * resultSize (2). + // Full matched index should be created. + executeQuery(query); + assertRemoteDocumentsRead(/* byKey= */ 0, /* byCollection= */ 2); + assertQueryReturned("coll/a", "coll/e"); + + setIndexAutoCreationEnabled(false); + + backfillIndexes(); + + applyRemoteEvent(addedRemoteEvent(doc("coll/f", 20, map("value", 7)), targetId)); + + executeQuery(query); + assertRemoteDocumentsRead(/* byKey= */ 2, /* byCollection= */ 1); + assertQueryReturned("coll/a", "coll/e", "coll/f"); + } + + @Test + public void testDisableIndexAutoCreationWorks() { + Query query1 = query("coll").filter(filter("value", "in", Arrays.asList(0, 1))); + int targetId1 = allocateQuery(query1); + + setIndexAutoCreationEnabled(true); + setMinCollectionSizeToAutoCreateIndex(0); + setRelativeIndexReadCostPerDocument(2); + + applyRemoteEvent(addedRemoteEvent(doc("coll/a", 10, map("value", 1)), targetId1)); + applyRemoteEvent(addedRemoteEvent(doc("coll/b", 10, map("value", 8)), targetId1)); + applyRemoteEvent(addedRemoteEvent(doc("coll/c", 10, map("value", "string")), targetId1)); + applyRemoteEvent(addedRemoteEvent(doc("coll/d", 10, map("value", false)), targetId1)); + applyRemoteEvent(addedRemoteEvent(doc("coll/e", 10, map("value", 0)), targetId1)); + + // First time query is running without indexes. + // Based on current heuristic, collection document counts (5) > 2 * resultSize (2). + // Full matched index should be created. + executeQuery(query1); + assertRemoteDocumentsRead(/* byKey= */ 0, /* byCollection= */ 2); + assertQueryReturned("coll/a", "coll/e"); + + setIndexAutoCreationEnabled(false); + + backfillIndexes(); + + executeQuery(query1); + assertRemoteDocumentsRead(/* byKey= */ 2, /* byCollection= */ 0); + assertQueryReturned("coll/a", "coll/e"); + + Query query2 = query("foo").filter(filter("value", "!=", Double.NaN)); + int targetId2 = allocateQuery(query2); + + applyRemoteEvent(addedRemoteEvent(doc("foo/a", 10, map("value", 5)), targetId2)); + applyRemoteEvent(addedRemoteEvent(doc("foo/b", 10, map("value", Double.NaN)), targetId2)); + applyRemoteEvent(addedRemoteEvent(doc("foo/c", 10, map("value", Double.NaN)), targetId2)); + applyRemoteEvent(addedRemoteEvent(doc("foo/d", 10, map("value", Double.NaN)), targetId2)); + applyRemoteEvent(addedRemoteEvent(doc("foo/e", 10, map("value", "string")), targetId2)); + + executeQuery(query2); + assertRemoteDocumentsRead(/* byKey= */ 0, /* byCollection= */ 2); + + backfillIndexes(); + + // Run the query in second time, test index won't be created + executeQuery(query2); + assertRemoteDocumentsRead(/* byKey= */ 0, /* byCollection= */ 2); + } + + @Test + public void testIndexAutoCreationWorksWithMutation() { + Query query = + query("coll").filter(filter("value", "array-contains-any", Arrays.asList(8, 1, "string"))); + int targetId = allocateQuery(query); + + setIndexAutoCreationEnabled(true); + setMinCollectionSizeToAutoCreateIndex(0); + setRelativeIndexReadCostPerDocument(2); + + applyRemoteEvent( + addedRemoteEvent(doc("coll/a", 10, map("value", Arrays.asList(8, 1, "string"))), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/b", 10, map("value", emptyList())), targetId)); + applyRemoteEvent(addedRemoteEvent(doc("coll/c", 10, map("value", singletonList(3))), targetId)); + applyRemoteEvent( + addedRemoteEvent(doc("coll/d", 10, map("value", Arrays.asList(0, 5))), targetId)); + applyRemoteEvent( + addedRemoteEvent(doc("coll/e", 10, map("value", singletonList("string"))), targetId)); + + executeQuery(query); + assertRemoteDocumentsRead(/* byKey= */ 0, /* byCollection= */ 2); + assertQueryReturned("coll/a", "coll/e"); + + writeMutation(deleteMutation("coll/e")); + + backfillIndexes(); + + writeMutation(setMutation("coll/f", map("value", singletonList(1)))); + + executeQuery(query); + assertRemoteDocumentsRead(/* byKey= */ 1, /* byCollection= */ 0); + assertOverlaysRead(/* byKey= */ 1, /* byCollection= */ 1); + assertQueryReturned("coll/a", "coll/f"); + } } diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/model/TargetIndexMatcherTest.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/model/TargetIndexMatcherTest.java index a71d18746fa..627775c8a2a 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/model/TargetIndexMatcherTest.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/model/TargetIndexMatcherTest.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue; import com.google.firebase.firestore.core.Query; +import com.google.firebase.firestore.core.Target; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -57,6 +58,14 @@ public class TargetIndexMatcherTest { query("collId") .filter(filter("a", "array-contains-any", Collections.singletonList("a")))); + List queriesWithOrderBy = + Arrays.asList( + query("collId").orderBy(orderBy("a")), + query("collId").orderBy(orderBy("a", "desc")), + query("collId").orderBy(orderBy("a", "asc")), + query("collId").orderBy(orderBy("a")).orderBy(orderBy("__name__")), + query("collId").filter(filter("a", "array-contains", "a")).orderBy(orderBy("b"))); + @Test public void canUseMergeJoin() { Query q = query("collId").filter(filter("a", "==", 1)).filter(filter("b", "==", 2)); @@ -632,4 +641,183 @@ private void validateDoesNotServeTarget( TargetIndexMatcher targetIndexMatcher = new TargetIndexMatcher(query.toTarget()); assertFalse(targetIndexMatcher.servedByIndex(expectedIndex)); } + + @Test + public void testBuildTargetIndexWithQueriesWithEqualities() { + for (Query query : queriesWithEqualities) { + validateBuildTargetIndexCreateFullMatchIndex(query); + } + } + + @Test + public void testBuildTargetIndexWithQueriesWithInequalities() { + for (Query query : queriesWithInequalities) { + validateBuildTargetIndexCreateFullMatchIndex(query); + } + } + + @Test + public void testBuildTargetIndexWithQueriesWithArrayContains() { + for (Query query : queriesWithArrayContains) { + validateBuildTargetIndexCreateFullMatchIndex(query); + } + } + + @Test + public void testBuildTargetIndexWithQueriesWithOrderBy() { + for (Query query : queriesWithOrderBy) { + validateBuildTargetIndexCreateFullMatchIndex(query); + } + } + + @Test + public void testBuildTargetIndexWithInequalityUsesSingleFieldIndex() { + Query query = query("collId").filter(filter("a", ">", 1)).filter(filter("a", "<", 10)); + validateBuildTargetIndexCreateFullMatchIndex(query); + } + + @Test + public void testBuildTargetIndexWithCollection() { + Query query = query("collId"); + validateBuildTargetIndexCreateFullMatchIndex(query); + } + + @Test + public void testBuildTargetIndexWithArrayContainsAndOrderBy() { + Query query = + query("collId") + .filter(filter("a", "array-contains", "a")) + .filter(filter("a", ">", "b")) + .orderBy(orderBy("a", "asc")); + validateBuildTargetIndexCreateFullMatchIndex(query); + } + + @Test + public void testBuildTargetIndexWithEqualityAndDescendingOrder() { + Query query = query("collId").filter(filter("a", "==", 1)).orderBy(orderBy("__name__", "desc")); + validateBuildTargetIndexCreateFullMatchIndex(query); + } + + @Test + public void testBuildTargetIndexWithMultipleEqualities() { + Query query = query("collId").filter(filter("a1", "==", "a")).filter(filter("a2", "==", "b")); + validateBuildTargetIndexCreateFullMatchIndex(query); + } + + @Test + public void testBuildTargetIndexWithMultipleEqualitiesAndInequality() { + Query query = + query("collId") + .filter(filter("equality1", "==", "a")) + .filter(filter("equality2", "==", "b")) + .filter(filter("inequality", ">=", "c")); + validateBuildTargetIndexCreateFullMatchIndex(query); + query = + query("collId") + .filter(filter("equality1", "==", "a")) + .filter(filter("inequality", ">=", "c")) + .filter(filter("equality2", "==", "b")); + validateBuildTargetIndexCreateFullMatchIndex(query); + } + + @Test + public void testBuildTargetIndexWithMultipleFilters() { + Query query = query("collId").filter(filter("a", "==", "a")).filter(filter("b", ">", "b")); + validateBuildTargetIndexCreateFullMatchIndex(query); + query = + query("collId") + .filter(filter("a1", "==", "a")) + .filter(filter("a2", ">", "b")) + .orderBy(orderBy("a2", "asc")); + validateBuildTargetIndexCreateFullMatchIndex(query); + query = + query("collId") + .filter(filter("a", ">=", 1)) + .filter(filter("a", "==", 5)) + .filter(filter("a", "<=", 10)); + validateBuildTargetIndexCreateFullMatchIndex(query); + query = + query("collId") + .filter(filter("a", "not-in", Arrays.asList(1, 2, 3))) + .filter(filter("a", ">=", 2)); + validateBuildTargetIndexCreateFullMatchIndex(query); + } + + @Test + public void testBuildTargetIndexWithMultipleOrderBys() { + Query query = + query("collId") + .orderBy(orderBy("fff")) + .orderBy(orderBy("bar", "desc")) + .orderBy(orderBy("__name__")); + validateBuildTargetIndexCreateFullMatchIndex(query); + query = + query("collId") + .orderBy(orderBy("foo")) + .orderBy(orderBy("bar")) + .orderBy(orderBy("__name__", "desc")); + validateBuildTargetIndexCreateFullMatchIndex(query); + } + + @Test + public void testBuildTargetIndexWithInAndNotIn() { + Query query = + query("collId") + .filter(filter("a", "not-in", Arrays.asList(1, 2, 3))) + .filter(filter("b", "in", Arrays.asList(1, 2, 3))); + validateBuildTargetIndexCreateFullMatchIndex(query); + } + + @Test + public void testBuildTargetIndexWithEqualityAndDifferentOrderBy() { + Query query = + query("collId") + .filter(filter("foo", "==", "")) + .filter(filter("bar", "==", "")) + .orderBy(orderBy("qux")); + validateBuildTargetIndexCreateFullMatchIndex(query); + query = + query("collId") + .filter(filter("aaa", "==", "")) + .filter(filter("qqq", "==", "")) + .filter(filter("ccc", "==", "")) + .orderBy(orderBy("fff", "desc")) + .orderBy(orderBy("bbb")); + validateBuildTargetIndexCreateFullMatchIndex(query); + } + + @Test + public void testBuildTargetIndexWithEqualsAndNotIn() { + Query query = + query("collId") + .filter(filter("a", "==", 1)) + .filter(filter("b", "not-in", Arrays.asList(1, 2, 3))); + validateBuildTargetIndexCreateFullMatchIndex(query); + } + + @Test + public void testBuildTargetIndexWithInAndOrderBy() { + Query query = + query("collId") + .filter(filter("a", "not-in", Arrays.asList(1, 2, 3))) + .orderBy(orderBy("a")) + .orderBy(orderBy("b")); + validateBuildTargetIndexCreateFullMatchIndex(query); + } + + @Test + public void testBuildTargetIndexWithInAndOrderBySameField() { + Query query = + query("collId").filter(filter("a", "in", Arrays.asList(1, 2, 3))).orderBy(orderBy("a")); + validateBuildTargetIndexCreateFullMatchIndex(query); + } + + private void validateBuildTargetIndexCreateFullMatchIndex(Query query) { + Target target = query.toTarget(); + TargetIndexMatcher targetIndexMatcher = new TargetIndexMatcher(target); + FieldIndex expectedIndex = targetIndexMatcher.buildTargetIndex(); + assertTrue(targetIndexMatcher.servedByIndex(expectedIndex)); + // Check the index created is a FULL MATCH index + assertTrue(expectedIndex.getSegments().size() >= target.getSegmentCount()); + } }