Skip to content

Commit 715f3f0

Browse files
schmidt-sebastianschmidt-sebastian
authored
Add Index-Free query engine (#697)
1 parent aafbec4 commit 715f3f0

File tree

20 files changed

+893
-60
lines changed

20 files changed

+893
-60
lines changed

‎firebase-database-collection/src/main/java/com/google/firebase/database/collection/ImmutableSortedMap.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,14 @@ public abstract class ImmutableSortedMap<K, V> implements Iterable<Map.Entry<K,
5656

5757
public abstract Comparator<K> getComparator();
5858

59+
public ImmutableSortedMap<K, V> insertAll(ImmutableSortedMap<K, V> source) {
60+
ImmutableSortedMap<K, V> result = this;
61+
for (Map.Entry<K, V> entry : source) {
62+
result = result.insert(entry.getKey(), entry.getValue());
63+
}
64+
return result;
65+
}
66+
5967
@Override
6068
@SuppressWarnings("unchecked")
6169
public boolean equals(Object o) {

‎firebase-firestore/src/androidTest/java/com/google/firebase/firestore/remote/RemoteStoreTest.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import com.google.firebase.firestore.local.LocalStore;
2525
import com.google.firebase.firestore.local.MemoryPersistence;
2626
import com.google.firebase.firestore.local.Persistence;
27+
import com.google.firebase.firestore.local.SimpleQueryEngine;
2728
import com.google.firebase.firestore.model.DocumentKey;
2829
import com.google.firebase.firestore.model.mutation.MutationBatchResult;
2930
import com.google.firebase.firestore.testutil.IntegrationTestUtil;
@@ -72,9 +73,10 @@ public ImmutableSortedSet<DocumentKey> getRemoteKeysForTarget(int targetId) {
7273
};
7374

7475
FakeConnectivityMonitor connectivityMonitor = new FakeConnectivityMonitor();
76+
SimpleQueryEngine queryEngine = new SimpleQueryEngine();
7577
Persistence persistence = MemoryPersistence.createEagerGcMemoryPersistence();
7678
persistence.start();
77-
LocalStore localStore = new LocalStore(persistence, User.UNAUTHENTICATED);
79+
LocalStore localStore = new LocalStore(persistence, queryEngine, User.UNAUTHENTICATED);
7880
RemoteStore remoteStore =
7981
new RemoteStore(callback, localStore, datastore, testQueue, connectivityMonitor);
8082

‎firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@
3737
import com.google.firebase.firestore.local.LruGarbageCollector;
3838
import com.google.firebase.firestore.local.MemoryPersistence;
3939
import com.google.firebase.firestore.local.Persistence;
40+
import com.google.firebase.firestore.local.QueryEngine;
4041
import com.google.firebase.firestore.local.SQLitePersistence;
42+
import com.google.firebase.firestore.local.SimpleQueryEngine;
4143
import com.google.firebase.firestore.model.Document;
4244
import com.google.firebase.firestore.model.DocumentKey;
4345
import com.google.firebase.firestore.model.MaybeDocument;
@@ -263,7 +265,9 @@ private void initialize(Context context, User user, boolean usePersistence, long
263265
}
264266

265267
persistence.start();
266-
localStore = new LocalStore(persistence, user);
268+
// TODO(index-free): Use IndexFreeQueryEngine/IndexedQueryEngine as appropriate.
269+
QueryEngine queryEngine = new SimpleQueryEngine();
270+
localStore = new LocalStore(persistence, queryEngine, user);
267271
if (gc != null) {
268272
lruScheduler = gc.newScheduler(asyncQueue, localStore);
269273
lruScheduler.start();

‎firebase-firestore/src/main/java/com/google/firebase/firestore/core/Query.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,18 @@ public boolean isCollectionGroupQuery() {
117117
return collectionGroup != null;
118118
}
119119

120+
/**
121+
* Returns true if this query does not specify any query constraints that could remove results.
122+
*/
123+
public boolean matchesAllDocuments() {
124+
return filters.isEmpty()
125+
&& limit == NO_LIMIT
126+
&& startAt == null
127+
&& endAt == null
128+
&& (getExplicitOrderBy().isEmpty()
129+
|| (getExplicitOrderBy().size() == 1 && getFirstOrderByField().isKeyField()));
130+
}
131+
120132
/** The filters on the documents returned by the query. */
121133
public List<Filter> getFilters() {
122134
return filters;

‎firebase-firestore/src/main/java/com/google/firebase/firestore/core/SyncEngine.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -193,9 +193,10 @@ public int listen(Query query) {
193193
private ViewSnapshot initializeViewAndComputeSnapshot(QueryData queryData) {
194194
Query query = queryData.getQuery();
195195

196-
ImmutableSortedMap<DocumentKey, Document> docs = localStore.executeQuery(query);
197196
ImmutableSortedSet<DocumentKey> remoteKeys =
198197
localStore.getRemoteDocumentKeys(queryData.getTargetId());
198+
ImmutableSortedMap<DocumentKey, Document> docs =
199+
localStore.executeQuery(query, queryData, remoteKeys);
199200

200201
View view = new View(query, remoteKeys);
201202
View.DocumentChanges viewDocChanges = view.computeDocChanges(docs);
@@ -557,7 +558,8 @@ private void emitNewSnapsAndNotifyLocalStore(
557558
// against the local store to make sure we didn't lose any good docs that had been past the
558559
// limit.
559560
ImmutableSortedMap<DocumentKey, Document> docs =
560-
localStore.executeQuery(queryView.getQuery());
561+
localStore.executeQuery(
562+
queryView.getQuery(), /* queryData= */ null, DocumentKey.emptyKeySet());
561563
viewDocChanges = view.computeDocChanges(docs, viewDocChanges);
562564
}
563565
TargetChange targetChange =

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexFreeQueryEngine.java

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
// Copyright 2019 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package com.google.firebase.firestore.local;
16+
17+
import static com.google.firebase.firestore.util.Assert.hardAssert;
18+
19+
import androidx.annotation.Nullable;
20+
import com.google.firebase.database.collection.ImmutableSortedMap;
21+
import com.google.firebase.database.collection.ImmutableSortedSet;
22+
import com.google.firebase.firestore.core.Query;
23+
import com.google.firebase.firestore.model.Document;
24+
import com.google.firebase.firestore.model.DocumentCollections;
25+
import com.google.firebase.firestore.model.DocumentKey;
26+
import com.google.firebase.firestore.model.MaybeDocument;
27+
import com.google.firebase.firestore.model.SnapshotVersion;
28+
import java.util.Map;
29+
30+
/**
31+
* A query engine that takes advantage of the target document mapping in the QueryCache. The
32+
* IndexFreeQueryEngine optimizes query execution by only reading the documents previously matched a
33+
* query plus any documents that were edited after the query was last listened to.
34+
*
35+
* <p>There are some cases where Index-Free queries are not guaranteed to produce to the same
36+
* results as full collection scans. In these case, the IndexFreeQueryEngine falls back to a full
37+
* query processing. These cases are:
38+
*
39+
* <ol>
40+
* <li>Limit queries where a document that matched the query previously no longer matches the
41+
* query. In this case, we have to scan all local documents since a document that was sent to
42+
* us as part of a different query result may now fall into the limit.
43+
* <li>Limit queries that include edits that occurred after the last remote snapshot (both
44+
* latency-compensated or committed). Even if an edited document continues to match the query,
45+
* an edit may cause a document to sort below another document that is in the local cache.
46+
* <li>Queries where the last snapshot contained Limbo documents. Even though a Limbo document is
47+
* not part of the backend result set, we need to include Limbo documents in local views to
48+
* ensure consistency between different Query views. If there exists a previous query snapshot
49+
* that contained no limbo documents, we can instead use the older snapshot version for
50+
* Index-Free processing.
51+
* </ol>
52+
*/
53+
public class IndexFreeQueryEngine implements QueryEngine {
54+
private LocalDocumentsView localDocumentsView;
55+
56+
@Override
57+
public void setLocalDocumentsView(LocalDocumentsView localDocuments) {
58+
this.localDocumentsView = localDocuments;
59+
}
60+
61+
@Override
62+
public ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingQuery(
63+
Query query, @Nullable QueryData queryData, ImmutableSortedSet<DocumentKey> remoteKeys) {
64+
hardAssert(localDocumentsView != null, "setLocalDocumentsView() not called");
65+
66+
// Queries that match all document don't benefit from using IndexFreeQueries. It is more
67+
// efficient to scan all documents in a collection, rather than to perform individual lookups.
68+
if (query.matchesAllDocuments()) {
69+
return executeFullCollectionScan(query);
70+
}
71+
72+
// Queries that have never seen a snapshot without limbo free documents should also be run as a
73+
// full collection scan.
74+
if (queryData == null
75+
|| queryData.getLastLimboFreeSnapshotVersion().equals(SnapshotVersion.NONE)) {
76+
return executeFullCollectionScan(query);
77+
}
78+
79+
ImmutableSortedMap<DocumentKey, Document> result =
80+
executeIndexFreeQuery(query, queryData, remoteKeys);
81+
82+
return result != null ? result : executeFullCollectionScan(query);
83+
}
84+
85+
/**
86+
* Attempts index-free query execution. Returns the set of query results on success, otherwise
87+
* returns null.
88+
*/
89+
private @Nullable ImmutableSortedMap<DocumentKey, Document> executeIndexFreeQuery(
90+
Query query, QueryData queryData, ImmutableSortedSet<DocumentKey> remoteKeys) {
91+
// Fetch the documents that matched the query at the last snapshot.
92+
ImmutableSortedMap<DocumentKey, MaybeDocument> previousResults =
93+
localDocumentsView.getDocuments(remoteKeys);
94+
95+
// Limit queries are not eligible for index-free query execution if any part of the result was
96+
// modified after we received the last query snapshot. This makes sure that we re-populate the
97+
// view with older documents that may sort before the modified document.
98+
if (query.hasLimit()
99+
&& containsUpdatesSinceSnapshotVersion(previousResults, queryData.getSnapshotVersion())) {
100+
return null;
101+
}
102+
103+
ImmutableSortedMap<DocumentKey, Document> results = DocumentCollections.emptyDocumentMap();
104+
105+
// Re-apply the query filter since previously matching documents do not necessarily still
106+
// match the query.
107+
for (Map.Entry<DocumentKey, MaybeDocument> entry : previousResults) {
108+
MaybeDocument maybeDoc = entry.getValue();
109+
if (maybeDoc instanceof Document && query.matches((Document) maybeDoc)) {
110+
Document doc = (Document) maybeDoc;
111+
results = results.insert(entry.getKey(), doc);
112+
} else if (query.hasLimit()) {
113+
// Limit queries with documents that no longer match need to be re-filled from cache.
114+
return null;
115+
}
116+
}
117+
118+
// Retrieve all results for documents that were updated since the last limbo-document free
119+
// remote snapshot.
120+
ImmutableSortedMap<DocumentKey, Document> updatedResults =
121+
localDocumentsView.getDocumentsMatchingQuery(
122+
query, queryData.getLastLimboFreeSnapshotVersion());
123+
124+
results = results.insertAll(updatedResults);
125+
126+
return results;
127+
}
128+
129+
@Override
130+
public void handleDocumentChange(MaybeDocument oldDocument, MaybeDocument newDocument) {
131+
// No indexes to update.
132+
}
133+
134+
private boolean containsUpdatesSinceSnapshotVersion(
135+
ImmutableSortedMap<DocumentKey, MaybeDocument> previousResults,
136+
SnapshotVersion sinceSnapshotVersion) {
137+
for (Map.Entry<DocumentKey, MaybeDocument> doc : previousResults) {
138+
if (doc.getValue().hasPendingWrites()
139+
|| doc.getValue().getVersion().compareTo(sinceSnapshotVersion) > 0) {
140+
return true;
141+
}
142+
}
143+
144+
return false;
145+
}
146+
147+
private ImmutableSortedMap<DocumentKey, Document> executeFullCollectionScan(Query query) {
148+
return localDocumentsView.getDocumentsMatchingQuery(query, SnapshotVersion.NONE);
149+
}
150+
}

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexedQueryEngine.java

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import androidx.annotation.Nullable;
2020
import androidx.annotation.VisibleForTesting;
2121
import com.google.firebase.database.collection.ImmutableSortedMap;
22+
import com.google.firebase.database.collection.ImmutableSortedSet;
2223
import com.google.firebase.firestore.core.FieldFilter;
2324
import com.google.firebase.firestore.core.Filter;
2425
import com.google.firebase.firestore.core.Filter.Operator;
@@ -88,17 +89,23 @@ public class IndexedQueryEngine implements QueryEngine {
8889
private static final List<Class> lowCardinalityTypes =
8990
Arrays.asList(BooleanValue.class, ArrayValue.class, ObjectValue.class);
9091

91-
private final LocalDocumentsView localDocuments;
9292
private final SQLiteCollectionIndex collectionIndex;
93+
private LocalDocumentsView localDocuments;
9394

94-
public IndexedQueryEngine(
95-
LocalDocumentsView localDocuments, SQLiteCollectionIndex collectionIndex) {
96-
this.localDocuments = localDocuments;
95+
public IndexedQueryEngine(SQLiteCollectionIndex collectionIndex) {
9796
this.collectionIndex = collectionIndex;
9897
}
9998

10099
@Override
101-
public ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingQuery(Query query) {
100+
public void setLocalDocumentsView(LocalDocumentsView localDocuments) {
101+
this.localDocuments = localDocuments;
102+
}
103+
104+
@Override
105+
public ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingQuery(
106+
Query query, @Nullable QueryData queryData, ImmutableSortedSet<DocumentKey> remoteKeys) {
107+
hardAssert(localDocuments != null, "setLocalDocumentsView() not called");
108+
102109
return query.isDocumentQuery()
103110
? localDocuments.getDocumentsMatchingQuery(query, SnapshotVersion.NONE)
104111
: performCollectionQuery(query);

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalDocumentsView.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
* mutations in the MutationQueue to the RemoteDocumentCache.
4141
*/
4242
// TODO: Turn this into the UnifiedDocumentCache / whatever.
43-
final class LocalDocumentsView {
43+
class LocalDocumentsView {
4444

4545
private final RemoteDocumentCache remoteDocumentCache;
4646
private final MutationQueue mutationQueue;

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import android.util.SparseArray;
2121
import androidx.annotation.Nullable;
22+
import androidx.annotation.VisibleForTesting;
2223
import com.google.firebase.Timestamp;
2324
import com.google.firebase.database.collection.ImmutableSortedMap;
2425
import com.google.firebase.database.collection.ImmutableSortedSet;
@@ -125,7 +126,7 @@ public final class LocalStore {
125126
/** Used to generate targetIds for queries tracked locally. */
126127
private final TargetIdGenerator targetIdGenerator;
127128

128-
public LocalStore(Persistence persistence, User initialUser) {
129+
public LocalStore(Persistence persistence, QueryEngine queryEngine, User initialUser) {
129130
hardAssert(
130131
persistence.isStarted(), "LocalStore was passed an unstarted persistence implementation");
131132
this.persistence = persistence;
@@ -135,8 +136,9 @@ public LocalStore(Persistence persistence, User initialUser) {
135136
remoteDocuments = persistence.getRemoteDocumentCache();
136137
localDocuments =
137138
new LocalDocumentsView(remoteDocuments, mutationQueue, persistence.getIndexManager());
138-
// TODO: Use IndexedQueryEngine as appropriate.
139-
queryEngine = new SimpleQueryEngine(localDocuments);
139+
140+
this.queryEngine = queryEngine;
141+
queryEngine.setLocalDocumentsView(localDocuments);
140142

141143
localViewReferences = new ReferenceSet();
142144
persistence.getReferenceDelegate().setInMemoryPins(localViewReferences);
@@ -170,8 +172,7 @@ public ImmutableSortedMap<DocumentKey, MaybeDocument> handleUserChange(User user
170172
// Recreate our LocalDocumentsView using the new MutationQueue.
171173
localDocuments =
172174
new LocalDocumentsView(remoteDocuments, mutationQueue, persistence.getIndexManager());
173-
// TODO: Use IndexedQueryEngine as appropriate.
174-
queryEngine = new SimpleQueryEngine(localDocuments);
175+
queryEngine.setLocalDocumentsView(localDocuments);
175176

176177
// Union the old/new changed keys.
177178
ImmutableSortedSet<DocumentKey> changedKeys = DocumentKey.emptyKeySet();
@@ -561,6 +562,21 @@ public QueryData allocateQuery(Query query) {
561562
return cached;
562563
}
563564

565+
/**
566+
* Returns the QueryData as seen by the LocalStore, including updates that may have not yet been
567+
* persisted to the QueryCache.
568+
*/
569+
@VisibleForTesting
570+
@Nullable
571+
QueryData getQueryData(Query query) {
572+
QueryData queryData = queryCache.getQueryData(query);
573+
if (queryData == null) {
574+
return null;
575+
}
576+
QueryData updatedQueryData = targetIds.get(queryData.getTargetId());
577+
return updatedQueryData != null ? updatedQueryData : queryData;
578+
}
579+
564580
/** Mutable state for the transaction in allocateQuery. */
565581
private static class AllocateQueryHolder {
566582
QueryData cached;
@@ -611,7 +627,23 @@ public void releaseQuery(Query query) {
611627

612628
/** Runs the given query against all the documents in the local store and returns the results. */
613629
public ImmutableSortedMap<DocumentKey, Document> executeQuery(Query query) {
614-
return queryEngine.getDocumentsMatchingQuery(query);
630+
QueryData queryData = getQueryData(query);
631+
if (queryData != null) {
632+
ImmutableSortedSet<DocumentKey> remoteKeys =
633+
this.queryCache.getMatchingKeysForTargetId(queryData.getTargetId());
634+
return executeQuery(query, queryData, remoteKeys);
635+
} else {
636+
return executeQuery(query, null, DocumentKey.emptyKeySet());
637+
}
638+
}
639+
640+
/**
641+
* Runs the given query against the local store and returns the results, potentially taking
642+
* advantage of the provided query data and the set of remote document keys.
643+
*/
644+
public ImmutableSortedMap<DocumentKey, Document> executeQuery(
645+
Query query, @Nullable QueryData queryData, ImmutableSortedSet<DocumentKey> remoteKeys) {
646+
return queryEngine.getDocumentsMatchingQuery(query, queryData, remoteKeys);
615647
}
616648

617649
/**

0 commit comments

Comments
 (0)