Skip to content

Commit d0048fd

Browse files
schmidt-sebastianschmidt-sebastian
authored
Use DocumentKey as tie breaker in index backfill (#3174)
1 parent ab6bab4 commit d0048fd

20 files changed

+288
-117
lines changed

‎firebase-crashlytics-ndk/src/third_party/crashpad

Submodule crashpad updated from 281ba70 to e5e47bc

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/DefaultQueryEngine.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.google.firebase.firestore.core.Query;
2222
import com.google.firebase.firestore.model.Document;
2323
import com.google.firebase.firestore.model.DocumentKey;
24+
import com.google.firebase.firestore.model.FieldIndex.IndexOffset;
2425
import com.google.firebase.firestore.model.SnapshotVersion;
2526
import com.google.firebase.firestore.util.Logger;
2627
import java.util.Collections;
@@ -98,7 +99,8 @@ && needsRefill(
9899
// Retrieve all results for documents that were updated since the last limbo-document free
99100
// remote snapshot.
100101
ImmutableSortedMap<DocumentKey, Document> updatedResults =
101-
localDocumentsView.getDocumentsMatchingQuery(query, lastLimboFreeSnapshotVersion);
102+
localDocumentsView.getDocumentsMatchingQuery(
103+
query, IndexOffset.create(lastLimboFreeSnapshotVersion));
102104

103105
// We merge `previousResults` into `updateResults`, since `updateResults` is already a
104106
// ImmutableSortedMap. If a document is contained in both lists, then its contents are the same.
@@ -168,6 +170,6 @@ private ImmutableSortedMap<DocumentKey, Document> executeFullCollectionScan(Quer
168170
if (Logger.isDebugEnabled()) {
169171
Logger.debug(LOG_TAG, "Using full collection scan to execute query: %s", query.toString());
170172
}
171-
return localDocumentsView.getDocumentsMatchingQuery(query, SnapshotVersion.NONE);
173+
return localDocumentsView.getDocumentsMatchingQuery(query, IndexOffset.NONE);
172174
}
173175
}

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexBackfiller.java

Lines changed: 33 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@
2323
import com.google.firebase.firestore.model.Document;
2424
import com.google.firebase.firestore.model.DocumentKey;
2525
import com.google.firebase.firestore.model.FieldIndex;
26+
import com.google.firebase.firestore.model.FieldIndex.IndexOffset;
2627
import com.google.firebase.firestore.model.ResourcePath;
27-
import com.google.firebase.firestore.model.SnapshotVersion;
2828
import com.google.firebase.firestore.util.AsyncQueue;
2929
import com.google.firebase.firestore.util.Logger;
3030
import java.util.ArrayList;
@@ -140,42 +140,53 @@ private int writeEntriesForCollectionGroup(
140140
LocalDocumentsView localDocumentsView, String collectionGroup, int entriesRemainingUnderCap) {
141141
Query query = new Query(ResourcePath.EMPTY, collectionGroup);
142142

143-
// Use the earliest readTime of all field indexes as the base readtime.
144-
SnapshotVersion earliestReadTime =
145-
getEarliestReadTime(indexManager.getFieldIndexes(collectionGroup));
143+
// Use the earliest offset of all field indexes to query the local cache.
144+
IndexOffset existingOffset = getExistingOffset(indexManager.getFieldIndexes(collectionGroup));
146145

147146
// TODO(indexing): Use limit queries to only fetch the required number of entries.
148147
// TODO(indexing): Support mutation batch Ids when sorting and writing indexes.
149148
ImmutableSortedMap<DocumentKey, Document> documents =
150-
localDocumentsView.getDocumentsMatchingQuery(query, earliestReadTime);
149+
localDocumentsView.getDocumentsMatchingQuery(query, existingOffset);
151150

152151
List<Document> oldestDocuments = getOldestDocuments(documents, entriesRemainingUnderCap);
153152
indexManager.updateIndexEntries(oldestDocuments);
154153

155-
SnapshotVersion latestReadTime = getPostUpdateReadTime(oldestDocuments, earliestReadTime);
156-
indexManager.updateCollectionGroup(collectionGroup, latestReadTime);
154+
IndexOffset newOffset = getNewOffset(oldestDocuments, existingOffset);
155+
indexManager.updateCollectionGroup(collectionGroup, newOffset);
157156
return oldestDocuments.size();
158157
}
159158

159+
/** Returns the lowest offset for the provided index group. */
160+
private IndexOffset getExistingOffset(Collection<FieldIndex> fieldIndexes) {
161+
IndexOffset lowestOffset = null;
162+
for (FieldIndex fieldIndex : fieldIndexes) {
163+
if (lowestOffset == null
164+
|| fieldIndex.getIndexState().getOffset().compareTo(lowestOffset) < 0) {
165+
lowestOffset = fieldIndex.getIndexState().getOffset();
166+
}
167+
}
168+
return lowestOffset == null ? IndexOffset.NONE : lowestOffset;
169+
}
170+
160171
/**
161-
* Returns the new read time for the index.
172+
* Returns the offset for the index based on the newly indexed documents.
162173
*
163-
* @param documents a list of documents sorted by read time (ascending)
164-
* @param currentReadTime the current read time of the index
174+
* @param documents a list of documents sorted by read time and key (ascending)
175+
* @param currentOffset the current offset of the index group
165176
*/
166-
private SnapshotVersion getPostUpdateReadTime(
167-
List<Document> documents, SnapshotVersion currentReadTime) {
168-
SnapshotVersion latestReadTime =
177+
private IndexOffset getNewOffset(List<Document> documents, IndexOffset currentOffset) {
178+
IndexOffset latestOffset =
169179
documents.isEmpty()
170-
? remoteDocumentCache.getLatestReadTime()
171-
: documents.get(documents.size() - 1).getReadTime();
180+
? IndexOffset.create(remoteDocumentCache.getLatestReadTime())
181+
: IndexOffset.create(
182+
documents.get(documents.size() - 1).getReadTime(),
183+
documents.get(documents.size() - 1).getKey());
172184
// Make sure the index does not go back in time
173-
latestReadTime =
174-
latestReadTime.compareTo(currentReadTime) > 0 ? latestReadTime : currentReadTime;
175-
return latestReadTime;
185+
latestOffset = latestOffset.compareTo(currentOffset) > 0 ? latestOffset : currentOffset;
186+
return latestOffset;
176187
}
177188

178-
/** Returns up to {@code count} documents sorted by read time. */
189+
/** Returns up to {@code count} documents sorted by read time and key. */
179190
private List<Document> getOldestDocuments(
180191
ImmutableSortedMap<DocumentKey, Document> documents, int count) {
181192
List<Document> oldestDocuments = new ArrayList<>();
@@ -184,27 +195,12 @@ private List<Document> getOldestDocuments(
184195
}
185196
Collections.sort(
186197
oldestDocuments,
187-
(l, r) -> {
188-
int cmp = l.getReadTime().compareTo(r.getReadTime());
189-
if (cmp != 0) return cmp;
190-
return l.getKey().compareTo(r.getKey());
191-
});
198+
(l, r) ->
199+
IndexOffset.create(l.getReadTime(), l.getKey())
200+
.compareTo(IndexOffset.create(r.getReadTime(), r.getKey())));
192201
return oldestDocuments.subList(0, Math.min(count, oldestDocuments.size()));
193202
}
194203

195-
private SnapshotVersion getEarliestReadTime(Collection<FieldIndex> fieldIndexes) {
196-
SnapshotVersion lowestVersion = null;
197-
for (FieldIndex fieldIndex : fieldIndexes) {
198-
lowestVersion =
199-
lowestVersion == null
200-
? fieldIndex.getIndexState().getReadTime()
201-
: fieldIndex.getIndexState().getReadTime().compareTo(lowestVersion) < 0
202-
? fieldIndex.getIndexState().getReadTime()
203-
: lowestVersion;
204-
}
205-
return lowestVersion;
206-
}
207-
208204
@VisibleForTesting
209205
void setMaxDocumentsToProcess(int newMax) {
210206
maxDocumentsToProcess = newMax;

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexManager.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import com.google.firebase.firestore.model.DocumentKey;
2121
import com.google.firebase.firestore.model.FieldIndex;
2222
import com.google.firebase.firestore.model.ResourcePath;
23-
import com.google.firebase.firestore.model.SnapshotVersion;
2423
import java.util.Collection;
2524
import java.util.List;
2625
import java.util.Set;
@@ -91,11 +90,11 @@ public interface IndexManager {
9190
/**
9291
* Sets the collection group's latest read time.
9392
*
94-
* <p>This method updates the read time for all field indices for the collection group and
93+
* <p>This method updates the index offset for all field indices for the collection group and
9594
* increments their sequence number. Subsequent calls to {@link #getNextCollectionGroupToUpdate()}
9695
* will return a different collection group (unless only one collection group is configured).
9796
*/
98-
void updateCollectionGroup(String collectionGroup, SnapshotVersion readTime);
97+
void updateCollectionGroup(String collectionGroup, FieldIndex.IndexOffset offset);
9998

10099
/** Updates the index entries for the provided documents. */
101100
void updateIndexEntries(Collection<Document> documents);

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexedQueryEngine.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ public ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingQuery(
5858
hardAssert(localDocuments != null, "setLocalDocumentsView() not called");
5959

6060
return query.isDocumentQuery()
61-
? localDocuments.getDocumentsMatchingQuery(query, SnapshotVersion.NONE)
61+
? localDocuments.getDocumentsMatchingQuery(query, FieldIndex.IndexOffset.NONE)
6262
: performCollectionQuery(query);
6363
}
6464

@@ -84,7 +84,7 @@ private ImmutableSortedMap<DocumentKey, Document> performCollectionQuery(Query q
8484
ImmutableSortedMap<DocumentKey, Document> indexedDocuments =
8585
localDocuments.getDocuments(keys);
8686
ImmutableSortedMap<DocumentKey, Document> additionalDocuments =
87-
localDocuments.getDocumentsMatchingQuery(query, fieldIndex.getIndexState().getReadTime());
87+
localDocuments.getDocumentsMatchingQuery(query, fieldIndex.getIndexState().getOffset());
8888
for (Map.Entry<DocumentKey, Document> entry : additionalDocuments) {
8989
indexedDocuments = indexedDocuments.insert(entry.getKey(), entry.getValue());
9090
}
@@ -98,6 +98,6 @@ private ImmutableSortedMap<DocumentKey, Document> executeFullCollectionScan(Quer
9898
if (Logger.isDebugEnabled()) {
9999
Logger.debug(LOG_TAG, "Using full collection scan to execute query: %s", query.toString());
100100
}
101-
return localDocuments.getDocumentsMatchingQuery(query, SnapshotVersion.NONE);
101+
return localDocuments.getDocumentsMatchingQuery(query, FieldIndex.IndexOffset.NONE);
102102
}
103103
}

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalDocumentsView.java

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@
2323
import com.google.firebase.firestore.core.Query;
2424
import com.google.firebase.firestore.model.Document;
2525
import com.google.firebase.firestore.model.DocumentKey;
26+
import com.google.firebase.firestore.model.FieldIndex.IndexOffset;
2627
import com.google.firebase.firestore.model.MutableDocument;
2728
import com.google.firebase.firestore.model.ResourcePath;
28-
import com.google.firebase.firestore.model.SnapshotVersion;
2929
import com.google.firebase.firestore.model.mutation.FieldMask;
3030
import com.google.firebase.firestore.model.mutation.Mutation;
3131
import com.google.firebase.firestore.model.mutation.MutationBatch;
@@ -246,18 +246,17 @@ void recalculateAndSaveOverlays(Set<DocumentKey> documentKeys) {
246246
* Performs a query against the local view of all documents.
247247
*
248248
* @param query The query to match documents against.
249-
* @param sinceReadTime If not set to SnapshotVersion.MIN, return only documents that have been
250-
* read since this snapshot version (exclusive).
249+
* @param offset Read time and key to start scanning by (exclusive).
251250
*/
252251
ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingQuery(
253-
Query query, SnapshotVersion sinceReadTime) {
252+
Query query, IndexOffset offset) {
254253
ResourcePath path = query.getPath();
255254
if (query.isDocumentQuery()) {
256255
return getDocumentsMatchingDocumentQuery(path);
257256
} else if (query.isCollectionGroupQuery()) {
258-
return getDocumentsMatchingCollectionGroupQuery(query, sinceReadTime);
257+
return getDocumentsMatchingCollectionGroupQuery(query, offset);
259258
} else {
260-
return getDocumentsMatchingCollectionQuery(query, sinceReadTime);
259+
return getDocumentsMatchingCollectionQuery(query, offset);
261260
}
262261
}
263262

@@ -274,7 +273,7 @@ private ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingDocumentQu
274273
}
275274

276275
private ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingCollectionGroupQuery(
277-
Query query, SnapshotVersion sinceReadTime) {
276+
Query query, IndexOffset offset) {
278277
hardAssert(
279278
query.getPath().isEmpty(),
280279
"Currently we only support collection group queries at the root.");
@@ -287,7 +286,7 @@ private ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingCollection
287286
for (ResourcePath parent : parents) {
288287
Query collectionQuery = query.asCollectionQueryAtPath(parent.append(collectionId));
289288
ImmutableSortedMap<DocumentKey, Document> collectionResults =
290-
getDocumentsMatchingCollectionQuery(collectionQuery, sinceReadTime);
289+
getDocumentsMatchingCollectionQuery(collectionQuery, offset);
291290
for (Map.Entry<DocumentKey, Document> docEntry : collectionResults) {
292291
results = results.insert(docEntry.getKey(), docEntry.getValue());
293292
}
@@ -296,11 +295,11 @@ private ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingCollection
296295
}
297296

298297
private ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingCollectionQuery(
299-
Query query, SnapshotVersion sinceReadTime) {
298+
Query query, IndexOffset offset) {
300299
if (Persistence.OVERLAY_SUPPORT_ENABLED) {
301300
// TODO(Overlay): Remove the assert and just return `fromOverlay`.
302301
ImmutableSortedMap<DocumentKey, Document> fromOverlay =
303-
getDocumentsMatchingCollectionQueryFromOverlayCache(query, sinceReadTime);
302+
getDocumentsMatchingCollectionQueryFromOverlayCache(query, offset);
304303
// TODO(Overlay): Delete below before merging. The code passes, but there are tests
305304
// looking at how many documents read from remote document, and this would double
306305
// the count.
@@ -313,16 +312,15 @@ private ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingCollection
313312
*/
314313
return fromOverlay;
315314
} else {
316-
return getDocumentsMatchingCollectionQueryFromMutationQueue(query, sinceReadTime);
315+
return getDocumentsMatchingCollectionQueryFromMutationQueue(query, offset);
317316
}
318317
}
319318

320319
/** Queries the remote documents and overlays by doing a full collection scan. */
321320
private ImmutableSortedMap<DocumentKey, Document>
322-
getDocumentsMatchingCollectionQueryFromOverlayCache(
323-
Query query, SnapshotVersion sinceReadTime) {
321+
getDocumentsMatchingCollectionQueryFromOverlayCache(Query query, IndexOffset offset) {
324322
ImmutableSortedMap<DocumentKey, MutableDocument> remoteDocuments =
325-
remoteDocumentCache.getAllDocumentsMatchingQuery(query, sinceReadTime);
323+
remoteDocumentCache.getAllDocumentsMatchingQuery(query, offset);
326324
Map<DocumentKey, Mutation> overlays = documentOverlayCache.getOverlays(query.getPath(), -1);
327325

328326
// As documents might match the query because of their overlay we need to include all documents
@@ -356,10 +354,9 @@ private ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingCollection
356354

357355
/** Queries the remote documents and mutation queue, by doing a full collection scan. */
358356
private ImmutableSortedMap<DocumentKey, Document>
359-
getDocumentsMatchingCollectionQueryFromMutationQueue(
360-
Query query, SnapshotVersion sinceReadTime) {
357+
getDocumentsMatchingCollectionQueryFromMutationQueue(Query query, IndexOffset offset) {
361358
ImmutableSortedMap<DocumentKey, MutableDocument> remoteDocuments =
362-
remoteDocumentCache.getAllDocumentsMatchingQuery(query, sinceReadTime);
359+
remoteDocumentCache.getAllDocumentsMatchingQuery(query, offset);
363360

364361
// TODO(indexing): We should plumb sinceReadTime through to the mutation queue
365362
List<MutationBatch> matchingBatches = mutationQueue.getAllMutationBatchesAffectingQuery(query);

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryIndexManager.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import com.google.firebase.firestore.model.DocumentKey;
2222
import com.google.firebase.firestore.model.FieldIndex;
2323
import com.google.firebase.firestore.model.ResourcePath;
24-
import com.google.firebase.firestore.model.SnapshotVersion;
2524
import java.util.ArrayList;
2625
import java.util.Collection;
2726
import java.util.Collections;
@@ -82,7 +81,7 @@ public String getNextCollectionGroupToUpdate() {
8281
}
8382

8483
@Override
85-
public void updateCollectionGroup(String collectionGroup, SnapshotVersion readTime) {
84+
public void updateCollectionGroup(String collectionGroup, FieldIndex.IndexOffset offset) {
8685
// Field indices are not supported with memory persistence.
8786
}
8887

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import com.google.firebase.database.collection.ImmutableSortedMap;
2323
import com.google.firebase.firestore.core.Query;
2424
import com.google.firebase.firestore.model.DocumentKey;
25+
import com.google.firebase.firestore.model.FieldIndex.IndexOffset;
2526
import com.google.firebase.firestore.model.MutableDocument;
2627
import com.google.firebase.firestore.model.ResourcePath;
2728
import com.google.firebase.firestore.model.SnapshotVersion;
@@ -83,7 +84,7 @@ public Map<DocumentKey, MutableDocument> getAll(Iterable<DocumentKey> keys) {
8384

8485
@Override
8586
public ImmutableSortedMap<DocumentKey, MutableDocument> getAllDocumentsMatchingQuery(
86-
Query query, SnapshotVersion sinceReadTime) {
87+
Query query, IndexOffset offset) {
8788
hardAssert(
8889
!query.isCollectionGroupQuery(),
8990
"CollectionGroup queries should be handled in LocalDocumentsView");
@@ -110,7 +111,7 @@ public ImmutableSortedMap<DocumentKey, MutableDocument> getAllDocumentsMatchingQ
110111
}
111112

112113
SnapshotVersion readTime = entry.getValue().second;
113-
if (readTime.compareTo(sinceReadTime) <= 0) {
114+
if (IndexOffset.create(readTime, doc.getKey()).compareTo(offset) <= 0) {
114115
continue;
115116
}
116117

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/RemoteDocumentCache.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import com.google.firebase.database.collection.ImmutableSortedMap;
1818
import com.google.firebase.firestore.core.Query;
1919
import com.google.firebase.firestore.model.DocumentKey;
20+
import com.google.firebase.firestore.model.FieldIndex;
2021
import com.google.firebase.firestore.model.MutableDocument;
2122
import com.google.firebase.firestore.model.SnapshotVersion;
2223
import java.util.Map;
@@ -73,12 +74,11 @@ interface RemoteDocumentCache {
7374
* <p>Cached entries for non-existing documents have no bearing on query results.
7475
*
7576
* @param query The query to match documents against.
76-
* @param sinceReadTime If not set to SnapshotVersion.MIN, return only documents that have been
77-
* read since this snapshot version (exclusive).
77+
* @param offset The read time and document key to start scanning at (exclusive).
7878
* @return The set of matching documents.
7979
*/
8080
ImmutableSortedMap<DocumentKey, MutableDocument> getAllDocumentsMatchingQuery(
81-
Query query, SnapshotVersion sinceReadTime);
81+
Query query, FieldIndex.IndexOffset offset);
8282

8383
/** Returns the latest read time of any document in the cache. */
8484
SnapshotVersion getLatestReadTime();

0 commit comments

Comments
 (0)