diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestore.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestore.java index 2acc26fe4ad..65d61ce490b 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestore.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestore.java @@ -220,12 +220,7 @@ private void ensureClientConfigured() { new DatabaseInfo(databaseId, persistenceKey, settings.getHost(), settings.isSslEnabled()); client = - new FirestoreClient( - context, - databaseInfo, - settings.isPersistenceEnabled(), - credentialsProvider, - asyncQueue); + new FirestoreClient(context, databaseInfo, settings, credentialsProvider, asyncQueue); } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestoreSettings.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestoreSettings.java index f0598e0b768..750e7ff0159 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestoreSettings.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestoreSettings.java @@ -24,6 +24,18 @@ /** Settings used to configure a FirebaseFirestore instance. */ @PublicApi public final class FirebaseFirestoreSettings { + /** + * Constant to use with {@link FirebaseFirestoreSettings.Builder#setCacheSizeBytes(long)} to + * disable garbage collection. + */ + @PublicApi public static final long CACHE_SIZE_UNLIMITED = -1; + + private static final long MINIMUM_CACHE_BYTES = 1 * 1024 * 1024; // 1 MB + // TODO(b/121269744): Set this to be the default value after SDK is past version 1.0 + // private static final long DEFAULT_CACHE_SIZE_BYTES = 100 * 1024 * 1024; // 100 MB + // For now, we are rolling this out with collection disabled. Once the SDK has hit version 1.0, + // we will switch the default to the above value, 100 MB. + private static final long DEFAULT_CACHE_SIZE_BYTES = CACHE_SIZE_UNLIMITED; private static final String DEFAULT_HOST = "firestore.googleapis.com"; private static final boolean DEFAULT_TIMESTAMPS_IN_SNAPSHOTS_ENABLED = false; @@ -34,6 +46,7 @@ public static final class Builder { private boolean sslEnabled; private boolean persistenceEnabled; private boolean timestampsInSnapshotsEnabled; + private long cacheSizeBytes; /** Constructs a new FirebaseFirestoreSettings Builder object. */ @PublicApi @@ -42,6 +55,7 @@ public Builder() { sslEnabled = true; persistenceEnabled = true; timestampsInSnapshotsEnabled = DEFAULT_TIMESTAMPS_IN_SNAPSHOTS_ENABLED; + cacheSizeBytes = DEFAULT_CACHE_SIZE_BYTES; } /** @@ -124,6 +138,30 @@ public Builder setTimestampsInSnapshotsEnabled(boolean value) { return this; } + /** + * Sets an approximate cache size threshold for the on-disk data. If the cache grows beyond this + * size, Firestore will start removing data that hasn't been recently used. The size is not a + * guarantee that the cache will stay below that size, only that if the cache exceeds the given + * size, cleanup will be attempted. + * + *

By default, collection is disabled (the value is set to {@link + * FirebaseFirestoreSettings#CACHE_SIZE_UNLIMITED}). In a future release, collection will be + * enabled by default, with a default cache size of 100 MB. The minimum value is 1 MB. + * + * @return A settings object on which the cache size is configured as specified by the given + * {@code value}. + */ + @NonNull + @PublicApi + public Builder setCacheSizeBytes(long value) { + if (value != CACHE_SIZE_UNLIMITED && value < MINIMUM_CACHE_BYTES) { + throw new IllegalArgumentException( + "Cache size must be set to at least " + MINIMUM_CACHE_BYTES + " bytes"); + } + this.cacheSizeBytes = value; + return this; + } + @NonNull @PublicApi public FirebaseFirestoreSettings build() { @@ -139,6 +177,7 @@ public FirebaseFirestoreSettings build() { private final boolean sslEnabled; private final boolean persistenceEnabled; private final boolean timestampsInSnapshotsEnabled; + private final long cacheSizeBytes; /** Constructs a FirebaseFirestoreSettings object based on the values in the Builder. */ private FirebaseFirestoreSettings(Builder builder) { @@ -146,6 +185,7 @@ private FirebaseFirestoreSettings(Builder builder) { sslEnabled = builder.sslEnabled; persistenceEnabled = builder.persistenceEnabled; timestampsInSnapshotsEnabled = builder.timestampsInSnapshotsEnabled; + cacheSizeBytes = builder.cacheSizeBytes; } @Override @@ -161,7 +201,8 @@ public boolean equals(@Nullable Object o) { return host.equals(that.host) && sslEnabled == that.sslEnabled && persistenceEnabled == that.persistenceEnabled - && timestampsInSnapshotsEnabled == that.timestampsInSnapshotsEnabled; + && timestampsInSnapshotsEnabled == that.timestampsInSnapshotsEnabled + && cacheSizeBytes == that.cacheSizeBytes; } @Override @@ -170,6 +211,7 @@ public int hashCode() { result = 31 * result + (sslEnabled ? 1 : 0); result = 31 * result + (persistenceEnabled ? 1 : 0); result = 31 * result + (timestampsInSnapshotsEnabled ? 1 : 0); + result = 31 * result + (int) cacheSizeBytes; return result; } @@ -211,4 +253,13 @@ public boolean isPersistenceEnabled() { public boolean areTimestampsInSnapshotsEnabled() { return timestampsInSnapshotsEnabled; } + + /** + * Returns the threshold for the cache size above which the SDK will attempt to collect the least + * recently used documents. + */ + @PublicApi + public long getCacheSizeBytes() { + return cacheSizeBytes; + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java index 9ea2b2f5b7d..123a4268a62 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java @@ -27,11 +27,14 @@ import com.google.firebase.firestore.EventListener; import com.google.firebase.firestore.FirebaseFirestoreException; import com.google.firebase.firestore.FirebaseFirestoreException.Code; +import com.google.firebase.firestore.FirebaseFirestoreSettings; import com.google.firebase.firestore.auth.CredentialsProvider; import com.google.firebase.firestore.auth.User; import com.google.firebase.firestore.core.EventManager.ListenOptions; import com.google.firebase.firestore.local.LocalSerializer; import com.google.firebase.firestore.local.LocalStore; +import com.google.firebase.firestore.local.LruDelegate; +import com.google.firebase.firestore.local.LruGarbageCollector; import com.google.firebase.firestore.local.MemoryPersistence; import com.google.firebase.firestore.local.Persistence; import com.google.firebase.firestore.local.SQLitePersistence; @@ -71,10 +74,13 @@ public final class FirestoreClient implements RemoteStore.RemoteStoreCallback { private SyncEngine syncEngine; private EventManager eventManager; + // LRU-related + @Nullable private LruGarbageCollector.Scheduler lruScheduler; + public FirestoreClient( final Context context, DatabaseInfo databaseInfo, - final boolean usePersistence, + FirebaseFirestoreSettings settings, CredentialsProvider credentialsProvider, final AsyncQueue asyncQueue) { this.databaseInfo = databaseInfo; @@ -105,7 +111,11 @@ public FirestoreClient( try { // Block on initial user being available User initialUser = Tasks.await(firstUser.getTask()); - initialize(context, initialUser, usePersistence); + initialize( + context, + initialUser, + settings.isPersistenceEnabled(), + settings.getCacheSizeBytes()); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } @@ -127,6 +137,9 @@ public Task shutdown() { () -> { remoteStore.shutdown(); persistence.shutdown(); + if (lruScheduler != null) { + lruScheduler.stop(); + } }); } @@ -194,24 +207,38 @@ public Task transaction( () -> syncEngine.transaction(asyncQueue, updateFunction, retries)); } - private void initialize(Context context, User user, boolean usePersistence) { + private void initialize(Context context, User user, boolean usePersistence, long cacheSizeBytes) { // Note: The initialization work must all be synchronous (we can't dispatch more work) since // external write/listen operations could get queued to run before that subsequent work // completes. Logger.debug(LOG_TAG, "Initializing. user=%s", user.getUid()); + LruGarbageCollector gc = null; if (usePersistence) { LocalSerializer serializer = new LocalSerializer(new RemoteSerializer(databaseInfo.getDatabaseId())); - persistence = + LruGarbageCollector.Params params = + LruGarbageCollector.Params.WithCacheSizeBytes(cacheSizeBytes); + SQLitePersistence sqlitePersistence = new SQLitePersistence( - context, databaseInfo.getPersistenceKey(), databaseInfo.getDatabaseId(), serializer); + context, + databaseInfo.getPersistenceKey(), + databaseInfo.getDatabaseId(), + serializer, + params); + LruDelegate lruDelegate = sqlitePersistence.getReferenceDelegate(); + gc = lruDelegate.getGarbageCollector(); + persistence = sqlitePersistence; } else { persistence = MemoryPersistence.createEagerGcMemoryPersistence(); } persistence.start(); localStore = new LocalStore(persistence, user); + if (gc != null) { + lruScheduler = gc.newScheduler(asyncQueue, localStore); + lruScheduler.start(); + } Datastore datastore = new Datastore(databaseInfo, asyncQueue, credentialsProvider, context); remoteStore = new RemoteStore(this, localStore, datastore, asyncQueue); diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java index 46d2d480b62..dae56aafe29 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java @@ -570,4 +570,8 @@ private void applyWriteToRemoteDocuments(MutationBatchResult batchResult) { mutationQueue.removeMutationBatch(batch); } + + public LruGarbageCollector.Results collectGarbage(LruGarbageCollector garbageCollector) { + return persistence.runTransaction("Collect garbage", () -> garbageCollector.collect(targetIds)); + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruDelegate.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruDelegate.java index 569ef9415ea..2dd128c285e 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruDelegate.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruDelegate.java @@ -21,12 +21,12 @@ * Persistence layers intending to use LRU Garbage collection should implement this interface. This * interface defines the operations that the LRU garbage collector needs from the persistence layer. */ -interface LruDelegate { +public interface LruDelegate { /** Enumerates all the targets in the QueryCache. */ void forEachTarget(Consumer consumer); - long getTargetCount(); + long getSequenceNumberCount(); /** Enumerates sequence numbers for documents not associated with a target. */ void forEachOrphanedDocumentSequenceNumber(Consumer consumer); @@ -49,4 +49,7 @@ interface LruDelegate { /** Access to the underlying LRU Garbage collector instance. */ LruGarbageCollector getGarbageCollector(); + + /** Return the size of the cache in bytes. */ + long getByteSize(); } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruGarbageCollector.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruGarbageCollector.java index 91a61731338..d423ca60218 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruGarbageCollector.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruGarbageCollector.java @@ -14,22 +14,156 @@ package com.google.firebase.firestore.local; +import android.support.annotation.Nullable; import android.util.SparseArray; +import com.google.firebase.firestore.FirebaseFirestoreSettings; import com.google.firebase.firestore.core.ListenSequence; +import com.google.firebase.firestore.util.AsyncQueue; +import com.google.firebase.firestore.util.Logger; import java.util.Comparator; +import java.util.Locale; import java.util.PriorityQueue; +import java.util.concurrent.TimeUnit; /** Implements the steps for LRU garbage collection. */ -class LruGarbageCollector { +public class LruGarbageCollector { + /** How long we wait to try running LRU GC after SDK initialization. */ + private static final long INITIAL_GC_DELAY_MS = TimeUnit.MINUTES.toMillis(1); + /** Minimum amount of time between GC checks, after the first one. */ + private static final long REGULAR_GC_DELAY_MS = TimeUnit.MINUTES.toMillis(5); + + public static class Params { + private static final long COLLECTION_DISABLED = FirebaseFirestoreSettings.CACHE_SIZE_UNLIMITED; + private static final long DEFAULT_CACHE_SIZE_BYTES = 100 * 1024 * 1024; // 100mb + /** + * The following two constants are estimates for how we want to tune the garbage collector. If + * we encounter a large cache, we don't want to spend a large chunk of time GCing all of it, we + * would rather make some progress and then try again later. We also don't want to collect + * everything that we possibly could, as our thesis is that recently used items are more likely + * to be used again. + */ + private static final int DEFAULT_COLLECTION_PERCENTILE = 10; + + private static final int DEFAULT_MAX_SEQUENCE_NUMBERS_TO_COLLECT = 1000; + + public static Params Default() { + return new Params( + DEFAULT_CACHE_SIZE_BYTES, + DEFAULT_COLLECTION_PERCENTILE, + DEFAULT_MAX_SEQUENCE_NUMBERS_TO_COLLECT); + } + + public static Params Disabled() { + return new Params(COLLECTION_DISABLED, 0, 0); + } + + public static Params WithCacheSizeBytes(long cacheSizeBytes) { + return new Params(cacheSizeBytes, 10, 1000); + } + + final long minBytesThreshold; + final int percentileToCollect; + final int maximumSequenceNumbersToCollect; + + Params(long minBytesThreshold, int percentileToCollect, int maximumSequenceNumbersToCollect) { + this.minBytesThreshold = minBytesThreshold; + this.percentileToCollect = percentileToCollect; + this.maximumSequenceNumbersToCollect = maximumSequenceNumbersToCollect; + } + } + + public static class Results { + private final boolean hasRun; + private final int sequenceNumbersCollected; + private final int targetsRemoved; + private final int documentsRemoved; + + static Results DidNotRun() { + return new Results(/* hasRun= */ false, 0, 0, 0); + } + + Results( + boolean hasRun, int sequenceNumbersCollected, int targetsRemoved, int documentsRemoved) { + this.hasRun = hasRun; + this.sequenceNumbersCollected = sequenceNumbersCollected; + this.targetsRemoved = targetsRemoved; + this.documentsRemoved = documentsRemoved; + } + + public boolean hasRun() { + return hasRun; + } + + public int getSequenceNumbersCollected() { + return sequenceNumbersCollected; + } + + public int getTargetsRemoved() { + return targetsRemoved; + } + + public int getDocumentsRemoved() { + return documentsRemoved; + } + } + + /** + * This class is responsible for the scheduling of LRU garbage collection. It handles checking + * whether or not GC is enabled, as well as which delay to use before the next run. + */ + public class Scheduler { + private final AsyncQueue asyncQueue; + private final LocalStore localStore; + private boolean hasRun = false; + @Nullable private AsyncQueue.DelayedTask gcTask; + + public Scheduler(AsyncQueue asyncQueue, LocalStore localStore) { + this.asyncQueue = asyncQueue; + this.localStore = localStore; + } + + public void start() { + if (params.minBytesThreshold != Params.COLLECTION_DISABLED) { + scheduleGC(); + } + } + + public void stop() { + if (gcTask != null) { + gcTask.cancel(); + } + } + + private void scheduleGC() { + long delay = hasRun ? REGULAR_GC_DELAY_MS : INITIAL_GC_DELAY_MS; + gcTask = + asyncQueue.enqueueAfterDelay( + AsyncQueue.TimerId.GARBAGE_COLLECTION, + delay, + () -> { + localStore.collectGarbage(LruGarbageCollector.this); + hasRun = true; + scheduleGC(); + }); + } + } + private final LruDelegate delegate; + private final Params params; - LruGarbageCollector(LruDelegate delegate) { + LruGarbageCollector(LruDelegate delegate, Params params) { this.delegate = delegate; + this.params = params; + } + + /** A helper method to create a new scheduler. */ + public Scheduler newScheduler(AsyncQueue asyncQueue, LocalStore localStore) { + return new Scheduler(asyncQueue, localStore); } /** Given a percentile of target to collect, returns the number of targets to collect. */ int calculateQueryCount(int percentile) { - long targetCount = delegate.getTargetCount(); + long targetCount = delegate.getSequenceNumberCount(); return (int) ((percentile / 100.0f) * targetCount); } @@ -66,7 +200,7 @@ long getMaxValue() { } /** Returns the nth sequence number, counting in order from the smallest. */ - long nthSequenceNumber(int count) { + long getNthSequenceNumber(int count) { if (count == 0) { return ListenSequence.INVALID; } @@ -91,4 +225,79 @@ int removeTargets(long upperBound, SparseArray activeTargetIds) { int removeOrphanedDocuments(long upperBound) { return delegate.removeOrphanedDocuments(upperBound); } + + Results collect(SparseArray activeTargetIds) { + if (params.minBytesThreshold == Params.COLLECTION_DISABLED) { + Logger.debug("LruGarbageCollector", "Garbage collection skipped; disabled"); + return Results.DidNotRun(); + } + + long cacheSize = getByteSize(); + if (cacheSize < params.minBytesThreshold) { + Logger.debug( + "LruGarbageCollector", + "Garbage collection skipped; Cache size " + + cacheSize + + " is lower than threshold " + + params.minBytesThreshold); + return Results.DidNotRun(); + } else { + return runGarbageCollection(activeTargetIds); + } + } + + private Results runGarbageCollection(SparseArray liveTargetIds) { + long startTs = System.currentTimeMillis(); + int sequenceNumbers = calculateQueryCount(params.percentileToCollect); + // Cap at the configured max + if (sequenceNumbers > params.maximumSequenceNumbersToCollect) { + Logger.debug( + "LruGarbageCollector", + "Capping sequence numbers to collect down to the maximum of " + + params.maximumSequenceNumbersToCollect + + " from " + + sequenceNumbers); + sequenceNumbers = params.maximumSequenceNumbersToCollect; + } + long countedTargetsTs = System.currentTimeMillis(); + + long upperBound = getNthSequenceNumber(sequenceNumbers); + long foundUpperBoundTs = System.currentTimeMillis(); + + int numTargetsRemoved = removeTargets(upperBound, liveTargetIds); + long removedTargetsTs = System.currentTimeMillis(); + + int numDocumentsRemoved = removeOrphanedDocuments(upperBound); + long removedDocumentsTs = System.currentTimeMillis(); + + if (Logger.isDebugEnabled()) { + String desc = "LRU Garbage Collection:\n"; + desc += "\tCounted targets in " + (countedTargetsTs - startTs) + "ms\n"; + desc += + String.format( + Locale.ROOT, + "\tDetermined least recently used %d sequence numbers in %dms\n", + sequenceNumbers, + (foundUpperBoundTs - countedTargetsTs)); + desc += + String.format( + Locale.ROOT, + "\tRemoved %d targets in %dms\n", + numTargetsRemoved, + (removedTargetsTs - foundUpperBoundTs)); + desc += + String.format( + Locale.ROOT, + "\tRemoved %d documents in %dms\n", + numDocumentsRemoved, + (removedDocumentsTs - removedTargetsTs)); + desc += String.format(Locale.ROOT, "Total Duration: %dms", (removedDocumentsTs - startTs)); + Logger.debug("LruGarbageCollector", desc); + } + return new Results(/* hasRun= */ true, sequenceNumbers, numTargetsRemoved, numDocumentsRemoved); + } + + long getByteSize() { + return delegate.getByteSize(); + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryLruReferenceDelegate.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryLruReferenceDelegate.java index 5376b5f9378..6ff4cb74dcc 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryLruReferenceDelegate.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryLruReferenceDelegate.java @@ -27,19 +27,24 @@ /** Provides LRU garbage collection functionality for MemoryPersistence. */ class MemoryLruReferenceDelegate implements ReferenceDelegate, LruDelegate { private final MemoryPersistence persistence; + private final LocalSerializer serializer; private final Map orphanedSequenceNumbers; private ReferenceSet inMemoryPins; private final LruGarbageCollector garbageCollector; private final ListenSequence listenSequence; private long currentSequenceNumber; - MemoryLruReferenceDelegate(MemoryPersistence persistence) { + MemoryLruReferenceDelegate( + MemoryPersistence persistence, + LruGarbageCollector.Params params, + LocalSerializer serializer) { this.persistence = persistence; + this.serializer = serializer; this.orphanedSequenceNumbers = new HashMap<>(); this.listenSequence = new ListenSequence(persistence.getQueryCache().getHighestListenSequenceNumber()); this.currentSequenceNumber = ListenSequence.INVALID; - this.garbageCollector = new LruGarbageCollector(this); + this.garbageCollector = new LruGarbageCollector(this, params); } @Override @@ -77,14 +82,24 @@ public void forEachTarget(Consumer consumer) { } @Override - public long getTargetCount() { - return persistence.getQueryCache().getTargetCount(); + public long getSequenceNumberCount() { + long targetCount = persistence.getQueryCache().getTargetCount(); + long orphanedCount[] = new long[1]; + forEachOrphanedDocumentSequenceNumber( + sequenceNumber -> { + orphanedCount[0]++; + }); + return targetCount + orphanedCount[0]; } @Override public void forEachOrphanedDocumentSequenceNumber(Consumer consumer) { - for (Long sequenceNumber : orphanedSequenceNumbers.values()) { - consumer.accept(sequenceNumber); + for (Map.Entry entry : orphanedSequenceNumbers.entrySet()) { + // Pass in the exact sequence number as the upper bound so we know it won't be pinned by being + // too recent. + if (!isPinned(entry.getKey(), entry.getValue())) { + consumer.accept(entry.getValue()); + } } } @@ -170,4 +185,18 @@ private boolean isPinned(DocumentKey key, long upperBound) { Long sequenceNumber = orphanedSequenceNumbers.get(key); return sequenceNumber != null && sequenceNumber > upperBound; } + + @Override + public long getByteSize() { + // Note that this method is only used for testing because this delegate is only + // used for testing. The algorithm here (loop through everything, serialize it + // and count bytes) is inefficient and inexact, but won't run in production. + long count = 0; + count += persistence.getQueryCache().getByteSize(serializer); + count += persistence.getRemoteDocumentCache().getByteSize(serializer); + for (MemoryMutationQueue queue : persistence.getMutationQueues()) { + count += queue.getByteSize(serializer); + } + return count; + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryMutationQueue.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryMutationQueue.java index 289bedf2348..05a666691dc 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryMutationQueue.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryMutationQueue.java @@ -371,4 +371,12 @@ private int indexOfExistingBatchId(int batchId, String action) { hardAssert(index >= 0 && index < queue.size(), "Batches must exist to be %s", action); return index; } + + long getByteSize(LocalSerializer serializer) { + long count = 0; + for (MutationBatch batch : queue) { + count += serializer.encodeMutationBatch(batch).getSerializedSize(); + } + return count; + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryPersistence.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryPersistence.java index 5b31f95a247..0f7698c4f03 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryPersistence.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryPersistence.java @@ -44,9 +44,11 @@ public static MemoryPersistence createEagerGcMemoryPersistence() { return persistence; } - public static MemoryPersistence createLruGcMemoryPersistence() { + public static MemoryPersistence createLruGcMemoryPersistence( + LruGarbageCollector.Params params, LocalSerializer serializer) { MemoryPersistence persistence = new MemoryPersistence(); - persistence.setReferenceDelegate(new MemoryLruReferenceDelegate(persistence)); + persistence.setReferenceDelegate( + new MemoryLruReferenceDelegate(persistence, params, serializer)); return persistence; } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryQueryCache.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryQueryCache.java index e37a451810d..7c0315dc3fe 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryQueryCache.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryQueryCache.java @@ -169,4 +169,12 @@ public ImmutableSortedSet getMatchingKeysForTargetId(int targetId) public boolean containsKey(DocumentKey key) { return references.containsKey(key); } + + long getByteSize(LocalSerializer serializer) { + long count = 0; + for (Map.Entry entry : queries.entrySet()) { + count += serializer.encodeQueryData(entry.getValue()).getSerializedSize(); + } + return count; + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java index 86c99a56b5f..9bed5c2dd39 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java @@ -100,4 +100,28 @@ public ImmutableSortedMap getAllDocumentsMatchingQuery(Qu ImmutableSortedMap getDocuments() { return docs; } + + /** + * Returns an estimate of the number of bytes used to store the given document key in memory. This + * is only an estimate and includes the size of the segments of the path, but not any object + * overhead or path separators. + */ + private static long getKeySize(DocumentKey key) { + ResourcePath path = key.getPath(); + long count = 0; + for (int i = 0; i < path.length(); i++) { + // Strings in java are utf-16, each character is two bytes in memory + count += path.getSegment(i).length() * 2; + } + return count; + } + + long getByteSize(LocalSerializer serializer) { + long count = 0; + for (Map.Entry entry : docs) { + count += getKeySize(entry.getKey()); + count += serializer.encodeMaybeDocument(entry.getValue()).getSerializedSize(); + } + return count; + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteLruReferenceDelegate.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteLruReferenceDelegate.java index cee90c741cc..d2823ac3990 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteLruReferenceDelegate.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteLruReferenceDelegate.java @@ -30,10 +30,10 @@ class SQLiteLruReferenceDelegate implements ReferenceDelegate, LruDelegate { private final LruGarbageCollector garbageCollector; private ReferenceSet inMemoryPins; - SQLiteLruReferenceDelegate(SQLitePersistence persistence) { + SQLiteLruReferenceDelegate(SQLitePersistence persistence, LruGarbageCollector.Params params) { this.currentSequenceNumber = ListenSequence.INVALID; this.persistence = persistence; - this.garbageCollector = new LruGarbageCollector(this); + this.garbageCollector = new LruGarbageCollector(this, params); } void start(long highestSequenceNumber) { @@ -70,8 +70,14 @@ public LruGarbageCollector getGarbageCollector() { } @Override - public long getTargetCount() { - return persistence.getQueryCache().getTargetCount(); + public long getSequenceNumberCount() { + long targetCount = persistence.getQueryCache().getTargetCount(); + long orphanedDocumentCount = + persistence + .query( + "SELECT COUNT(*) FROM (SELECT sequence_number FROM target_documents GROUP BY path HAVING COUNT(*) = 1 AND target_id = 0)") + .firstValue(row -> row.getLong(0)); + return targetCount + orphanedDocumentCount; } @Override @@ -179,4 +185,9 @@ private void writeSentinel(DocumentKey key) { path, getCurrentSequenceNumber()); } + + @Override + public long getByteSize() { + return persistence.getByteSize(); + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java index 7ecf6050c52..52799912639 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java @@ -34,6 +34,7 @@ import com.google.firebase.firestore.util.Consumer; import com.google.firebase.firestore.util.Logger; import com.google.firebase.firestore.util.Supplier; +import java.io.File; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.ArrayList; @@ -76,6 +77,7 @@ public static String databaseName(String persistenceKey, DatabaseId databaseId) private final OpenHelper opener; private final LocalSerializer serializer; private SQLiteDatabase db; + private File databasePath; private boolean started; private final SQLiteQueryCache queryCache; private final SQLiteRemoteDocumentCache remoteDocumentCache; @@ -97,13 +99,18 @@ public void onRollback() {} }; public SQLitePersistence( - Context context, String persistenceKey, DatabaseId databaseId, LocalSerializer serializer) { + Context context, + String persistenceKey, + DatabaseId databaseId, + LocalSerializer serializer, + LruGarbageCollector.Params params) { String databaseName = databaseName(persistenceKey, databaseId); this.opener = new OpenHelper(context, databaseName); + this.databasePath = context.getDatabasePath(databaseName); this.serializer = serializer; this.queryCache = new SQLiteQueryCache(this, this.serializer); this.remoteDocumentCache = new SQLiteRemoteDocumentCache(this, this.serializer); - this.referenceDelegate = new SQLiteLruReferenceDelegate(this); + this.referenceDelegate = new SQLiteLruReferenceDelegate(this, params); } @Override @@ -142,7 +149,7 @@ public boolean isStarted() { } @Override - public ReferenceDelegate getReferenceDelegate() { + public SQLiteLruReferenceDelegate getReferenceDelegate() { return referenceDelegate; } @@ -191,6 +198,10 @@ T runTransaction(String action, Supplier operation) { return value; } + long getByteSize() { + return databasePath.length(); + } + /** * A SQLiteOpenHelper that configures database connections just the way we like them, delegating * to SQLiteSchema to actually do the work of migration. diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/util/AsyncQueue.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/util/AsyncQueue.java index bef4ee4b12f..4954c14dc70 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/util/AsyncQueue.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/util/AsyncQueue.java @@ -68,6 +68,8 @@ public enum TimerId { * set timeout, rather than waiting indefinitely for success or failure. */ ONLINE_STATE_TIMEOUT, + /** A timer used to periodically attempt LRU Garbage collection */ + GARBAGE_COLLECTION } /** diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LruGarbageCollectorTestCase.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LruGarbageCollectorTestCase.java index 11cbc6a3493..65bdab22b17 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LruGarbageCollectorTestCase.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LruGarbageCollectorTestCase.java @@ -55,12 +55,13 @@ public abstract class LruGarbageCollectorTestCase { private MutationQueue mutationQueue; private RemoteDocumentCache documentCache; private LruGarbageCollector garbageCollector; + private LruGarbageCollector.Params lruParams; private int previousTargetId; private int previousDocNum; private long initialSequenceNumber; private ObjectValue testValue; - abstract Persistence createPersistence(); + abstract Persistence createPersistence(LruGarbageCollector.Params params); @Before public void setUp() { @@ -81,7 +82,11 @@ public void tearDown() { } private void newTestResources() { - persistence = createPersistence(); + newTestResources(LruGarbageCollector.Params.Default()); + } + + private void newTestResources(LruGarbageCollector.Params params) { + persistence = createPersistence(params); persistence.getReferenceDelegate().setInMemoryPins(new ReferenceSet()); queryCache = persistence.getQueryCache(); documentCache = persistence.getRemoteDocumentCache(); @@ -89,6 +94,7 @@ private void newTestResources() { mutationQueue = persistence.getMutationQueue(user); initialSequenceNumber = queryCache.getHighestListenSequenceNumber(); garbageCollector = ((LruDelegate) persistence.getReferenceDelegate()).getGarbageCollector(); + lruParams = params; } private QueryData nextQueryData() { @@ -193,7 +199,7 @@ public void testPickSequenceNumberPercentile() { @Test public void testSequenceNumberNoQueries() { - assertEquals(ListenSequence.INVALID, garbageCollector.nthSequenceNumber(0)); + assertEquals(ListenSequence.INVALID, garbageCollector.getNthSequenceNumber(0)); } @Test @@ -203,7 +209,7 @@ public void testSequenceNumberForFiftyQueries() { for (int i = 0; i < 50; i++) { addNextQuery(); } - assertEquals(initialSequenceNumber + 10, garbageCollector.nthSequenceNumber(10)); + assertEquals(initialSequenceNumber + 10, garbageCollector.getNthSequenceNumber(10)); } @Test @@ -220,7 +226,7 @@ public void testSequenceNumberForMultipleQueriesInATransaction() { for (int i = 9; i < 50; i++) { addNextQuery(); } - assertEquals(2 + initialSequenceNumber, garbageCollector.nthSequenceNumber(10)); + assertEquals(2 + initialSequenceNumber, garbageCollector.getNthSequenceNumber(10)); } @Test @@ -240,7 +246,7 @@ public void testAllCollectedQueriesInSingleTransaction() { for (int i = 11; i < 50; i++) { addNextQuery(); } - assertEquals(1 + initialSequenceNumber, garbageCollector.nthSequenceNumber(10)); + assertEquals(1 + initialSequenceNumber, garbageCollector.getNthSequenceNumber(10)); } @Test @@ -251,7 +257,7 @@ public void testSequenceNumbersWithMutationAndSequentialQueries() { for (int i = 0; i < 50; i++) { addNextQuery(); } - assertEquals(10 + initialSequenceNumber, garbageCollector.nthSequenceNumber(10)); + assertEquals(10 + initialSequenceNumber, garbageCollector.getNthSequenceNumber(10)); } @Test @@ -279,7 +285,7 @@ public void testSequenceNumbersWithMutationsInQueries() { addDocumentToTarget(docInQuery, queryData.getTargetId()); }); // This should catch the remaining 8 documents, plus the first two queries we added. - assertEquals(3 + initialSequenceNumber, garbageCollector.nthSequenceNumber(10)); + assertEquals(3 + initialSequenceNumber, garbageCollector.getNthSequenceNumber(10)); } @Test @@ -587,4 +593,105 @@ public void testRemoveTargetsThenGC() { } }); } + + @Test + public void testGetsSize() { + long initialSize = garbageCollector.getByteSize(); + + persistence.runTransaction( + "fill cache", + () -> { + // Simulate a bunch of ack'd mutations + for (int i = 0; i < 50; i++) { + Document doc = cacheADocumentInTransaction(); + markDocumentEligibleForGcInTransaction(doc.getKey()); + } + }); + + long finalSize = garbageCollector.getByteSize(); + assertTrue(finalSize > initialSize); + } + + @Test + public void testDisabled() { + LruGarbageCollector.Params params = LruGarbageCollector.Params.Disabled(); + + // Switch out the test resources for ones with a disabled GC. + persistence.shutdown(); + newTestResources(params); + + persistence.runTransaction( + "Fill cache", + () -> { + // Simulate a bunch of ack'd mutations + for (int i = 0; i < 500; i++) { + Document doc = cacheADocumentInTransaction(); + markDocumentEligibleForGcInTransaction(doc.getKey()); + } + }); + + LruGarbageCollector.Results results = + persistence.runTransaction("GC", () -> garbageCollector.collect(new SparseArray<>())); + + assertFalse(results.hasRun()); + } + + @Test + public void testCacheTooSmall() { + // Default LRU Params are ok for this test. + + persistence.runTransaction( + "Fill cache", + () -> { + // Simulate a bunch of ack'd mutations + for (int i = 0; i < 50; i++) { + Document doc = cacheADocumentInTransaction(); + markDocumentEligibleForGcInTransaction(doc.getKey()); + } + }); + + // Make sure we're under the target size + long cacheSize = garbageCollector.getByteSize(); + assertTrue(cacheSize < lruParams.minBytesThreshold); + + LruGarbageCollector.Results results = + persistence.runTransaction("GC", () -> garbageCollector.collect(new SparseArray<>())); + + assertFalse(results.hasRun()); + } + + @Test + public void testGCRan() { + // Set a low byte threshold so we can guarantee that GC will run. + LruGarbageCollector.Params params = LruGarbageCollector.Params.WithCacheSizeBytes(100); + + // Switch to persistence using our new params. + persistence.shutdown(); + newTestResources(params); + + // Add 100 targets and 10 documents to each + for (int i = 0; i < 100; i++) { + // Use separate transactions so that each target and associated documents get their own + // sequence number. + persistence.runTransaction( + "Add a target and some documents", + () -> { + QueryData queryData = addNextQueryInTransaction(); + for (int j = 0; j < 10; j++) { + Document doc = cacheADocumentInTransaction(); + addDocumentToTarget(doc.getKey(), queryData.getTargetId()); + } + }); + } + + // Mark nothing as live, so everything is eligible. + LruGarbageCollector.Results results = + persistence.runTransaction("GC", () -> garbageCollector.collect(new SparseArray<>())); + + // By default, we collect 10% of the sequence numbers. Since we added 100 targets, + // that should be 10 targets with 10 documents each, for a total of 100 documents. + assertTrue(results.hasRun()); + assertEquals(10, results.getTargetsRemoved()); + assertEquals(100, results.getDocumentsRemoved()); + } } diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/MemoryLruGarbageCollectorTest.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/MemoryLruGarbageCollectorTest.java index 98074286836..52ed3f456b9 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/MemoryLruGarbageCollectorTest.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/MemoryLruGarbageCollectorTest.java @@ -22,7 +22,7 @@ @Config(manifest = Config.NONE) public class MemoryLruGarbageCollectorTest extends LruGarbageCollectorTestCase { @Override - Persistence createPersistence() { - return PersistenceTestHelpers.createLRUMemoryPersistence(); + Persistence createPersistence(LruGarbageCollector.Params params) { + return PersistenceTestHelpers.createLRUMemoryPersistence(params); } } diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/PersistenceTestHelpers.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/PersistenceTestHelpers.java index e3dfb2669e5..f816937667b 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/PersistenceTestHelpers.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/PersistenceTestHelpers.java @@ -25,10 +25,16 @@ public final class PersistenceTestHelpers { private static int databaseNameCounter = 0; public static SQLitePersistence openSQLitePersistence(String name) { + return openSQLitePersistence(name, LruGarbageCollector.Params.Default()); + } + + public static SQLitePersistence openSQLitePersistence( + String name, LruGarbageCollector.Params params) { DatabaseId databaseId = DatabaseId.forProject("projectId"); LocalSerializer serializer = new LocalSerializer(new RemoteSerializer(databaseId)); Context context = RuntimeEnvironment.application; - SQLitePersistence persistence = new SQLitePersistence(context, name, databaseId, serializer); + SQLitePersistence persistence = + new SQLitePersistence(context, name, databaseId, serializer, params); persistence.start(); return persistence; } @@ -43,10 +49,14 @@ public static String nextSQLiteDatabaseName() { * @return a new SQLitePersistence with an empty database and an up-to-date schema. */ public static SQLitePersistence createSQLitePersistence() { + return createSQLitePersistence(LruGarbageCollector.Params.Default()); + } + + public static SQLitePersistence createSQLitePersistence(LruGarbageCollector.Params params) { // Robolectric's test runner will clear out the application database directory in between test // cases, but sometimes (particularly the spec tests) we create multiple databases per test // case and each should be fresh. A unique name is sufficient to keep these separate. - return openSQLitePersistence(nextSQLiteDatabaseName()); + return openSQLitePersistence(nextSQLiteDatabaseName(), params); } /** Creates and starts a new MemoryPersistence instance for testing. */ @@ -57,7 +67,14 @@ public static MemoryPersistence createEagerGCMemoryPersistence() { } public static MemoryPersistence createLRUMemoryPersistence() { - MemoryPersistence persistence = MemoryPersistence.createLruGcMemoryPersistence(); + return createLRUMemoryPersistence(LruGarbageCollector.Params.Default()); + } + + public static MemoryPersistence createLRUMemoryPersistence(LruGarbageCollector.Params params) { + DatabaseId databaseId = DatabaseId.forProject("projectId"); + LocalSerializer serializer = new LocalSerializer(new RemoteSerializer(databaseId)); + MemoryPersistence persistence = + MemoryPersistence.createLruGcMemoryPersistence(params, serializer); persistence.start(); return persistence; } diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteLruGarbageCollectorTest.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteLruGarbageCollectorTest.java index 792cce7c096..e3e2838eeff 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteLruGarbageCollectorTest.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteLruGarbageCollectorTest.java @@ -22,7 +22,7 @@ @Config(manifest = Config.NONE) public class SQLiteLruGarbageCollectorTest extends LruGarbageCollectorTestCase { @Override - Persistence createPersistence() { - return PersistenceTestHelpers.createSQLitePersistence(); + Persistence createPersistence(LruGarbageCollector.Params params) { + return PersistenceTestHelpers.createSQLitePersistence(params); } } diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/spec/SQLiteSpecTest.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/spec/SQLiteSpecTest.java index 011215b7454..c8940dbc14e 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/spec/SQLiteSpecTest.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/spec/SQLiteSpecTest.java @@ -14,6 +14,7 @@ package com.google.firebase.firestore.spec; +import com.google.firebase.firestore.local.LruGarbageCollector; import com.google.firebase.firestore.local.Persistence; import com.google.firebase.firestore.local.PersistenceTestHelpers; import java.util.Set; @@ -43,7 +44,8 @@ protected void specTearDown() throws Exception { @Override Persistence getPersistence(boolean garbageCollectionEnabled) { - return PersistenceTestHelpers.openSQLitePersistence(databaseName); + return PersistenceTestHelpers.openSQLitePersistence( + databaseName, LruGarbageCollector.Params.Default()); } @Override