Skip to content

Add getOverlays(CollectionGroup) #3330

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jan 19, 2022
Merged

Add getOverlays(CollectionGroup) #3330

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9d64048
Add Overlay class
schmidt-sebastian Jan 19, 2022
076af38
Add getOverlays(CollectionGroup)
schmidt-sebastian Jan 19, 2022
97faab1
Merge
schmidt-sebastian Jan 19, 2022
01f0fdf
Fix
schmidt-sebastian Jan 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions ...ase-firestore/src/main/java/com/google/firebase/firestore/local/DocumentOverlayCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import com.google.firebase.firestore.model.DocumentKey;
import com.google.firebase.firestore.model.ResourcePath;
import com.google.firebase.firestore.model.mutation.Mutation;
import com.google.firebase.firestore.model.mutation.Overlay;
import java.util.Map;

/**
Expand All @@ -35,7 +36,7 @@ public interface DocumentOverlayCache {
* for that key.
*/
@Nullable
Mutation getOverlay(DocumentKey key);
Overlay getOverlay(DocumentKey key);

/**
* Saves the given document key to mutation map to persistence as overlays. All overlays will have
Expand All @@ -52,6 +53,19 @@ public interface DocumentOverlayCache {
* @param collection The collection path to get the overlays for.
* @param sinceBatchId The minimum batch ID to filter by (exclusive). Only overlays that contain a
* change past `sinceBatchId` are returned.
* @return Mapping of each document key in the collection to its overlay.
*/
Map<DocumentKey, Mutation> getOverlays(ResourcePath collection, int sinceBatchId);
Map<DocumentKey, Overlay> getOverlays(ResourcePath collection, int sinceBatchId);

/**
* Returns {@code count} overlays with a batch ID higher than {@code sinceBatchId} for the
* provided collection group, processed by ascending batch ID. The method always returns all
* overlays for a batch even if a batch contains more documents than the remaining limit.
*
* @param collectionGroup The collection group to get the overlays for.
* @param sinceBatchId The minimum batch ID to filter by (exclusive). Only overlays that contain a
* change past `sinceBatchId` are returned.
* @return Mapping of each document key in the collection group to its overlay.
*/
Map<DocumentKey, Overlay> getOverlays(String collectionGroup, int sinceBatchId, int count);
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

optional: include count as @param

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

}
22 changes: 11 additions & 11 deletions firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalDocumentsView.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import com.google.firebase.firestore.model.mutation.FieldMask;
import com.google.firebase.firestore.model.mutation.Mutation;
import com.google.firebase.firestore.model.mutation.MutationBatch;
import com.google.firebase.firestore.model.mutation.Overlay;
import com.google.firebase.firestore.model.mutation.PatchMutation;
import java.util.HashMap;
import java.util.HashSet;
Expand All @@ -42,7 +43,6 @@
* in remoteDocumentCache or local mutations for the document). The view is computed by applying the
* mutations in the MutationQueue to the RemoteDocumentCache.
*/
// TODO: Turn this into the UnifiedDocumentCache / whatever.
class LocalDocumentsView {

private final RemoteDocumentCache remoteDocumentCache;
Expand Down Expand Up @@ -88,14 +88,14 @@ IndexManager getIndexManager() {
* for it.
*/
Document getDocument(DocumentKey key) {
Mutation overlay = documentOverlayCache.getOverlay(key);
Overlay overlay = documentOverlayCache.getOverlay(key);
// Only read from remote document cache if overlay is a patch.
MutableDocument document =
(overlay == null || overlay instanceof PatchMutation)
(overlay == null || overlay.getMutation() instanceof PatchMutation)
? remoteDocumentCache.get(key)
: MutableDocument.newInvalidDocument(key);
if (overlay != null) {
overlay.applyToLocalView(document, null, Timestamp.now());
overlay.getMutation().applyToLocalView(document, null, Timestamp.now());
}

return document;
Expand Down Expand Up @@ -125,18 +125,18 @@ ImmutableSortedMap<DocumentKey, Document> getLocalViewOfDocuments(
ImmutableSortedMap<DocumentKey, Document> results = emptyDocumentMap();
Map<DocumentKey, MutableDocument> recalculateDocuments = new HashMap<>();
for (Map.Entry<DocumentKey, MutableDocument> entry : docs.entrySet()) {
Mutation overlay = documentOverlayCache.getOverlay(entry.getKey());
Overlay overlay = documentOverlayCache.getOverlay(entry.getKey());
// Recalculate an overlay if the document's existence state is changed due to a remote
// event *and* the overlay is a PatchMutation. This is because document existence state
// can change if some patch mutation's preconditions are met.
// NOTE: we recalculate when `overlay` is null as well, because there might be a patch
// mutation whose precondition does not match before the change (hence overlay==null),
// but would now match.
if (existenceStateChanged.contains(entry.getKey())
&& (overlay == null || overlay instanceof PatchMutation)) {
&& (overlay == null || overlay.getMutation() instanceof PatchMutation)) {
recalculateDocuments.put(entry.getKey(), docs.get(entry.getKey()));
} else if (overlay != null) {
overlay.applyToLocalView(entry.getValue(), null, Timestamp.now());
overlay.getMutation().applyToLocalView(entry.getValue(), null, Timestamp.now());
}
}

Expand Down Expand Up @@ -260,11 +260,11 @@ private ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingCollection
Query query, IndexOffset offset) {
Map<DocumentKey, MutableDocument> remoteDocuments =
remoteDocumentCache.getAll(query.getPath(), offset);
Map<DocumentKey, Mutation> overlays = documentOverlayCache.getOverlays(query.getPath(), -1);
Map<DocumentKey, Overlay> overlays = documentOverlayCache.getOverlays(query.getPath(), -1);

// As documents might match the query because of their overlay we need to include documents
// for all overlays in the initial document set.
for (Map.Entry<DocumentKey, Mutation> entry : overlays.entrySet()) {
for (Map.Entry<DocumentKey, Overlay> entry : overlays.entrySet()) {
if (!remoteDocuments.containsKey(entry.getKey())) {
remoteDocuments.put(entry.getKey(), MutableDocument.newInvalidDocument(entry.getKey()));
}
Expand All @@ -273,9 +273,9 @@ private ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingCollection
// Apply the overlays and match against the query.
ImmutableSortedMap<DocumentKey, Document> results = emptyDocumentMap();
for (Map.Entry<DocumentKey, MutableDocument> docEntry : remoteDocuments.entrySet()) {
Mutation overlay = overlays.get(docEntry.getKey());
Overlay overlay = overlays.get(docEntry.getKey());
if (overlay != null) {
overlay.applyToLocalView(docEntry.getValue(), null, Timestamp.now());
overlay.getMutation().applyToLocalView(docEntry.getValue(), null, Timestamp.now());
}
// Finally, insert the documents that still match the query
if (query.matches(docEntry.getValue())) {
Expand Down
65 changes: 46 additions & 19 deletions ...restore/src/main/java/com/google/firebase/firestore/local/MemoryDocumentOverlayCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,31 +14,28 @@

package com.google.firebase.firestore.local;

import android.util.Pair;
import androidx.annotation.Nullable;
import com.google.firebase.firestore.model.DocumentKey;
import com.google.firebase.firestore.model.ResourcePath;
import com.google.firebase.firestore.model.mutation.Mutation;
import com.google.firebase.firestore.model.mutation.Overlay;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;

public class MemoryDocumentOverlayCache implements DocumentOverlayCache {
// A map sorted by DocumentKey, whose value is a pair of the largest batch id for the overlay
// and the overlay itself.
private final TreeMap<DocumentKey, Pair<Integer, Mutation>> overlays = new TreeMap<>();
private final TreeMap<DocumentKey, Overlay> overlays = new TreeMap<>();
private final Map<Integer, Set<DocumentKey>> overlayByBatchId = new HashMap<>();

@Nullable
@Override
public Mutation getOverlay(DocumentKey key) {
Pair<Integer, Mutation> overlay = overlays.get(key);
if (overlay != null) {
return overlay.second;
}
return null;
public Overlay getOverlay(DocumentKey key) {
return overlays.get(key);
}

private void saveOverlay(int largestBatchId, @Nullable Mutation mutation) {
Expand All @@ -47,12 +44,12 @@ private void saveOverlay(int largestBatchId, @Nullable Mutation mutation) {
}

// Remove the association of the overlay to its batch id.
Pair<Integer, Mutation> existing = this.overlays.get(mutation.getKey());
Overlay existing = this.overlays.get(mutation.getKey());
if (existing != null) {
overlayByBatchId.get(existing.first).remove(mutation.getKey());
overlayByBatchId.get(existing.getLargestBatchId()).remove(mutation.getKey());
}

overlays.put(mutation.getKey(), new Pair<>(largestBatchId, mutation));
overlays.put(mutation.getKey(), Overlay.create(largestBatchId, mutation));

// Create the associate of this overlay to the given largestBatchId.
if (overlayByBatchId.get(largestBatchId) == null) {
Expand Down Expand Up @@ -80,15 +77,15 @@ public void removeOverlaysForBatchId(int batchId) {
}

@Override
public Map<DocumentKey, Mutation> getOverlays(ResourcePath collection, int sinceBatchId) {
Map<DocumentKey, Mutation> result = new HashMap<>();
public Map<DocumentKey, Overlay> getOverlays(ResourcePath collection, int sinceBatchId) {
Map<DocumentKey, Overlay> result = new HashMap<>();

int immediateChildrenPathLength = collection.length() + 1;
DocumentKey prefix = DocumentKey.fromPath(collection.append(""));
Map<DocumentKey, Pair<Integer, Mutation>> view = overlays.tailMap(prefix);
Map<DocumentKey, Overlay> view = overlays.tailMap(prefix);

for (Map.Entry<DocumentKey, Pair<Integer, Mutation>> entry : view.entrySet()) {
DocumentKey key = entry.getKey();
for (Overlay overlay : view.values()) {
DocumentKey key = overlay.getKey();
if (!collection.isPrefixOf(key.getPath())) {
break;
}
Expand All @@ -97,9 +94,39 @@ public Map<DocumentKey, Mutation> getOverlays(ResourcePath collection, int since
continue;
}

Pair<Integer, Mutation> batchIdToOverlay = entry.getValue();
if (batchIdToOverlay.first > sinceBatchId) {
result.put(entry.getKey(), batchIdToOverlay.second);
if (overlay.getLargestBatchId() > sinceBatchId) {
result.put(overlay.getKey(), overlay);
}
}

return result;
}

@Override
public Map<DocumentKey, Overlay> getOverlays(
String collectionGroup, int sinceBatchId, int count) {
SortedMap<Integer, Map<DocumentKey, Overlay>> batchIdToOverlays = new TreeMap<>();

for (Overlay overlay : overlays.values()) {
DocumentKey key = overlay.getKey();
if (!key.getCollectionGroup().equals(collectionGroup)) {
continue;
}
if (overlay.getLargestBatchId() > sinceBatchId) {
Map<DocumentKey, Overlay> overlays = batchIdToOverlays.get(overlay.getLargestBatchId());
if (overlays == null) {
overlays = new HashMap<>();
batchIdToOverlays.put(overlay.getLargestBatchId(), overlays);
}
overlays.put(overlay.getKey(), overlay);
}
}

Map<DocumentKey, Overlay> result = new HashMap<>();
for (Map<DocumentKey, Overlay> overlays : batchIdToOverlays.values()) {
result.putAll(overlays);
if (result.size() >= count) {
break;
}
}

Expand Down
92 changes: 66 additions & 26 deletions ...restore/src/main/java/com/google/firebase/firestore/local/SQLiteDocumentOverlayCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.google.firebase.firestore.model.DocumentKey;
import com.google.firebase.firestore.model.ResourcePath;
import com.google.firebase.firestore.model.mutation.Mutation;
import com.google.firebase.firestore.model.mutation.Overlay;
import com.google.firestore.v1.Write;
import com.google.protobuf.InvalidProtocolBufferException;
import java.util.HashMap;
Expand All @@ -39,23 +40,14 @@ public SQLiteDocumentOverlayCache(SQLitePersistence db, LocalSerializer serializ

@Nullable
@Override
public Mutation getOverlay(DocumentKey key) {
public Overlay getOverlay(DocumentKey key) {
String collectionPath = EncodedPath.encode(key.getPath().popLast());
String documentId = key.getPath().getLastSegment();
return db.query(
"SELECT overlay_mutation FROM document_overlays WHERE uid = ? AND collection_path = ? AND document_id = ?")
"SELECT overlay_mutation, largest_batch_id FROM document_overlays "
+ "WHERE uid = ? AND collection_path = ? AND document_id = ?")
.binding(uid, collectionPath, documentId)
.firstValue(
row -> {
if (row == null) return null;

try {
Write mutation = Write.parseFrom(row.getBlob(0));
return serializer.decodeMutation(mutation);
} catch (InvalidProtocolBufferException e) {
throw fail("Overlay failed to parse: %s", e);
}
});
.firstValue(this::decodeOverlay);
}

private void saveOverlay(int largestBatchId, DocumentKey key, @Nullable Mutation mutation) {
Expand Down Expand Up @@ -90,28 +82,76 @@ public void removeOverlaysForBatchId(int batchId) {
}

@Override
public Map<DocumentKey, Mutation> getOverlays(ResourcePath collection, int sinceBatchId) {
public Map<DocumentKey, Overlay> getOverlays(ResourcePath collection, int sinceBatchId) {
String collectionPath = EncodedPath.encode(collection);

Map<DocumentKey, Mutation> result = new HashMap<>();

Map<DocumentKey, Overlay> result = new HashMap<>();
db.query(
"SELECT document_id, overlay_mutation FROM document_overlays "
"SELECT overlay_mutation, largest_batch_id FROM document_overlays "
+ "WHERE uid = ? AND collection_path = ? AND largest_batch_id > ?")
.binding(uid, collectionPath, sinceBatchId)
.forEach(
row -> {
try {
String documentId = row.getString(0);
Write write = Write.parseFrom(row.getBlob(1));
Mutation mutation = serializer.decodeMutation(write);

result.put(DocumentKey.fromPath(collection.append(documentId)), mutation);
} catch (InvalidProtocolBufferException e) {
throw fail("Overlay failed to parse: %s", e);
}
Overlay overlay = decodeOverlay(row);
result.put(overlay.getKey(), overlay);
});

return result;
}

@Override
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Love this implementation. So much cleaner!

public Map<DocumentKey, Overlay> getOverlays(
String collectionGroup, int sinceBatchId, int count) {
Map<DocumentKey, Overlay> result = new HashMap<>();
Overlay[] lastOverlay = new Overlay[] {null};

db.query(
"SELECT overlay_mutation, largest_batch_id FROM document_overlays "
+ "WHERE uid = ? AND collection_group = ? AND largest_batch_id > ? "
+ "ORDER BY largest_batch_id, collection_path, document_id LIMIT ?")
.binding(uid, collectionGroup, sinceBatchId, count)
.forEach(
row -> {
lastOverlay[0] = decodeOverlay(row);
result.put(lastOverlay[0].getKey(), lastOverlay[0]);
});

if (lastOverlay[0] == null) {
return result;
}

// Finish batch
DocumentKey key = lastOverlay[0].getKey();
String encodedCollectionPath = EncodedPath.encode(key.getCollectionPath());
db.query(
"SELECT overlay_mutation, largest_batch_id FROM document_overlays "
+ "WHERE uid = ? AND collection_group = ? "
+ "AND (collection_path > ? OR (collection_path = ? AND document_id > ?)) "
+ "AND largest_batch_id = ?")
.binding(
uid,
collectionGroup,
encodedCollectionPath,
encodedCollectionPath,
key.getDocumentId(),
lastOverlay[0].getLargestBatchId())
.forEach(
row -> {
Overlay overlay = decodeOverlay(row);
result.put(overlay.getKey(), overlay);
});

return result;
}

private Overlay decodeOverlay(android.database.Cursor row) {
try {
Write write = Write.parseFrom(row.getBlob(0));
int largestBatchId = row.getInt(1);
Mutation mutation = serializer.decodeMutation(write);
return Overlay.create(largestBatchId, mutation);
} catch (InvalidProtocolBufferException e) {
throw fail("Overlay failed to parse: %s", e);
}
}
}
2 changes: 1 addition & 1 deletion firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteSchema.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ private void removeMutationBatch(String uid, int batchId) {
mutationDeleter.bindString(1, uid);
mutationDeleter.bindLong(2, batchId);
int deleted = mutationDeleter.executeUpdateDelete();
hardAssert(deleted != 0, "Mutatiohn batch (%s, %d) did not exist", uid, batchId);
hardAssert(deleted != 0, "Mutation batch (%s, %d) did not exist", uid, batchId);

// Delete all index entries for this batch
db.execSQL(
Expand Down
Loading