Wire together LRU garbage collection

packages/firebase/index.d.ts

@@ -742,6 +742,8 @@ declare namespace firebase.firestore {
    */
   export type UpdateData = { [fieldPath: string]: any };
 
+  export const CACHE_SIZE_UNLIMITED: number;
+
   /** Settings used to configure a `Firestore` instance. */
   export interface Settings {
     /** The hostname to connect to. */
@@ -768,6 +770,17 @@ declare namespace firebase.firestore {
      * use Timestamp now and opt-in to this new behavior as soon as you can.
      */
     timestampsInSnapshots?: boolean;
+
+    /**
+     * An approximate cache size threshold for the on-disk data. If the cache grows beyond this
+     * size, Firestore will start removing data that hasn't been recently used. The size is not a
+     * guarantee that the cache will stay below that size, only that if the cache exceeds the given
+     * size, cleanup will be attempted.
+     *
+     * The default value is 40 MB. The threshold must be set to at least 1 MB, and can be set to
+     * CACHE_SIZE_UNLIMITED to disable garbage collection.
+     */
+    cacheSizeBytes?: number;
   }
 
   /**

packages/firestore/src/api/database.ts

@@ -32,6 +32,7 @@ import {
 } from '../core/query';
 import { Transaction as InternalTransaction } from '../core/transaction';
 import { ChangeType, ViewSnapshot } from '../core/view_snapshot';
+import { LruParams } from '../local/lru_garbage_collector';
 import { Document, MaybeDocument, NoDocument } from '../model/document';
 import { DocumentKey } from '../model/document_key';
 import {
@@ -101,6 +102,10 @@ import {
 const DEFAULT_HOST = 'firestore.googleapis.com';
 const DEFAULT_SSL = true;
 const DEFAULT_TIMESTAMPS_IN_SNAPSHOTS = false;
+const MINIMUM_CACHE_BYTES = 1 * 1024 * 1024;
+const DEFAULT_CACHE_SIZE_BYTES = LruParams.DEFAULT_CACHE_SIZE_BYTES;
+
+export const CACHE_SIZE_UNLIMITED = LruParams.COLLECTION_DISABLED;
 
 // enablePersistence() defaults:
 const DEFAULT_SYNCHRONIZE_TABS = false;
@@ -127,12 +132,14 @@ export interface FirestoreDatabase {
  */
 class FirestoreSettings {
   /** The hostname to connect to. */
-  host: string;
+  readonly host: string;
 
   /** Whether to use SSL when connecting. */
-  ssl: boolean;
+  readonly ssl: boolean;
+
+  readonly timestampsInSnapshots: boolean;
 
-  timestampsInSnapshots: boolean;
+  readonly cacheSizeBytes: number;
 
   // Can be a google-auth-library or gapi client.
   // tslint:disable-next-line:no-any
@@ -159,7 +166,8 @@ class FirestoreSettings {
       'host',
       'ssl',
       'credentials',
-      'timestampsInSnapshots'
+      'timestampsInSnapshots',
+      'cacheSizeBytes'
     ]);
 
     validateNamedOptionalType(
@@ -180,14 +188,34 @@ class FirestoreSettings {
       settings.timestampsInSnapshots,
       DEFAULT_TIMESTAMPS_IN_SNAPSHOTS
     );
+
+    validateNamedOptionalType(
+      'settings',
+      'number',
+      'cacheSizeBytes',
+      settings.cacheSizeBytes
+    );
+    if (settings.cacheSizeBytes === undefined) {
+      this.cacheSizeBytes = DEFAULT_CACHE_SIZE_BYTES;
+    } else {
+      if (settings.cacheSizeBytes < MINIMUM_CACHE_BYTES) {
+        throw new FirestoreError(
+          Code.INVALID_ARGUMENT,
+          `cacheSizeBytes must be at least ${MINIMUM_CACHE_BYTES}`
+        );
+      } else {
+        this.cacheSizeBytes = settings.cacheSizeBytes;
+      }
+    }
   }
 
   isEqual(other: FirestoreSettings): boolean {
     return (
       this.host === other.host &&
       this.ssl === other.ssl &&
       this.timestampsInSnapshots === other.timestampsInSnapshots &&
-      this.credentials === other.credentials
+      this.credentials === other.credentials &&
+      this.cacheSizeBytes === other.cacheSizeBytes
     );
   }
 }
@@ -418,7 +446,10 @@ follow these steps, YOUR APP MAY BREAK.`);
       this._config.credentials,
       this._queue
     );
-    return this._firestoreClient.start(persistenceSettings);
+    return this._firestoreClient.start(
+      persistenceSettings,
+      this._config.settings.cacheSizeBytes
+    );
   }
 
   private static databaseIdFromApp(app: FirebaseApp): DatabaseId {

packages/firestore/src/core/firestore_client.ts

@@ -46,6 +46,11 @@ import { SyncEngine } from './sync_engine';
 import { View, ViewDocumentChanges } from './view';
 
 import { PersistenceSettings } from '../api/database';
+import {
+  LruGarbageCollector,
+  LruParams,
+  LruScheduler
+} from '../local/lru_garbage_collector';
 import {
   MemorySharedClientState,
   SharedClientState,
@@ -84,6 +89,7 @@ export class FirestoreClient {
   private localStore: LocalStore;
   private remoteStore: RemoteStore;
   private syncEngine: SyncEngine;
+  private lruScheduler?: LruScheduler;
 
   private readonly clientId = AutoId.newId();
 
@@ -140,7 +146,10 @@ export class FirestoreClient {
    *     start for any reason. If usePersistence is false this is
    *     unconditionally resolved.
    */
-  start(persistenceSettings: PersistenceSettings): Promise<void> {
+  start(
+    persistenceSettings: PersistenceSettings,
+    cacheSizeBytes: number
+  ): Promise<void> {
     // We defer our initialization until we get the current user from
     // setChangeListener(). We block the async queue until we got the initial
     // user and the initialization is completed. This will prevent any scheduled
@@ -163,8 +172,13 @@ export class FirestoreClient {
       if (!initialized) {
         initialized = true;
 
-        this.initializePersistence(persistenceSettings, persistenceResult, user)
-          .then(() => this.initializeRest(user))
+        this.initializePersistence(
+          persistenceSettings,
+          persistenceResult,
+          user,
+          cacheSizeBytes
+        )
+          .then(maybeLruGc => this.initializeRest(user, maybeLruGc))
           .then(initializationDone.resolve, initializationDone.reject);
       } else {
         this.asyncQueue.enqueueAndForget(() => {
@@ -211,19 +225,27 @@ export class FirestoreClient {
   private initializePersistence(
     persistenceSettings: PersistenceSettings,
     persistenceResult: Deferred<void>,
-    user: User
-  ): Promise<void> {
+    user: User,
+    cacheSizeBytes: number
+  ): Promise<LruGarbageCollector | null> {
     if (persistenceSettings.enabled) {
-      return this.startIndexedDbPersistence(user, persistenceSettings)
-        .then(persistenceResult.resolve)
+      return this.startIndexedDbPersistence(
+        user,
+        persistenceSettings,
+        cacheSizeBytes
+      )
+        .then(maybeLruGc => {
+          persistenceResult.resolve();
+          return maybeLruGc;
+        })
         .catch(error => {
           // Regardless of whether or not the retry succeeds, from an user
           // perspective, offline persistence has failed.
           persistenceResult.reject(error);
 
           // An unknown failure on the first stage shuts everything down.
           if (!this.canFallback(error)) {
-            return Promise.reject(error);
+            throw error;
           }
 
           console.warn(
@@ -279,8 +301,9 @@ export class FirestoreClient {
    */
   private startIndexedDbPersistence(
     user: User,
-    settings: PersistenceSettings
-  ): Promise<void> {
+    settings: PersistenceSettings,
+    cacheSizeBytes: number
+  ): Promise<LruGarbageCollector> {
     assert(
       settings.enabled,
       'Should only start IndexedDb persitence with offline persistence enabled.'
@@ -307,6 +330,8 @@ export class FirestoreClient {
         );
       }
 
+      let persistence: IndexedDbPersistence;
+      const lruParams = LruParams.withCacheSize(cacheSizeBytes);
       if (settings.experimentalTabSynchronization) {
         this.sharedClientState = new WebStorageSharedClientState(
           this.asyncQueue,
@@ -315,24 +340,28 @@ export class FirestoreClient {
           this.clientId,
           user
         );
-        this.persistence = await IndexedDbPersistence.createMultiClientIndexedDbPersistence(
+        persistence = await IndexedDbPersistence.createMultiClientIndexedDbPersistence(
           storagePrefix,
           this.clientId,
           this.platform,
           this.asyncQueue,
           serializer,
+          lruParams,
           { sequenceNumberSyncer: this.sharedClientState }
         );
       } else {
         this.sharedClientState = new MemorySharedClientState();
-        this.persistence = await IndexedDbPersistence.createIndexedDbPersistence(
+        persistence = await IndexedDbPersistence.createIndexedDbPersistence(
           storagePrefix,
           this.clientId,
           this.platform,
           this.asyncQueue,
-          serializer
+          serializer,
+          lruParams
         );
       }
+      this.persistence = persistence;
+      return persistence.referenceDelegate.garbageCollector;
     });
   }
 
@@ -341,30 +370,35 @@ export class FirestoreClient {
    *
    * @returns A promise that will successfully resolve.
    */
-  private startMemoryPersistence(): Promise<void> {
-    // Opt to use proto3 JSON in case the platform doesn't support Uint8Array.
-    const serializer = new JsonProtoSerializer(this.databaseInfo.databaseId, {
-      useProto3Json: true
-    });
-    this.persistence = MemoryPersistence.createEagerPersistence(
-      this.clientId,
-      serializer
-    );
+  private startMemoryPersistence(): Promise<LruGarbageCollector | null> {
+    this.persistence = MemoryPersistence.createEagerPersistence(this.clientId);
     this.sharedClientState = new MemorySharedClientState();
-    return Promise.resolve();
+    return Promise.resolve(null);
   }
 
   /**
    * Initializes the rest of the FirestoreClient, assuming the initial user
    * has been obtained from the credential provider and some persistence
    * implementation is available in this.persistence.
    */
-  private initializeRest(user: User): Promise<void> {
+  private initializeRest(
+    user: User,
+    maybeLruGc: LruGarbageCollector | null
+  ): Promise<void> {
     debug(LOG_TAG, 'Initializing. user=', user.uid);
     return this.platform
       .loadConnection(this.databaseInfo)
       .then(async connection => {
         this.localStore = new LocalStore(this.persistence, user);
+        if (maybeLruGc) {
+          // We're running LRU Garbage collection. Set up the scheduler.
+          this.lruScheduler = new LruScheduler(
+            maybeLruGc,
+            this.asyncQueue,
+            this.localStore
+          );
+          this.lruScheduler.start();
+        }
         const serializer = this.platform.newSerializer(
           this.databaseInfo.databaseId
         );
@@ -439,6 +473,9 @@ export class FirestoreClient {
   }): Promise<void> {
     return this.asyncQueue.enqueue(async () => {
       // PORTING NOTE: LocalStore does not need an explicit shutdown on web.
+      if (this.lruScheduler) {
+        this.lruScheduler.stop();
+      }
       await this.remoteStore.shutdown();
       await this.sharedClientState.shutdown();
       await this.persistence.shutdown(

packages/firestore/src/local/indexeddb_persistence.ts

@@ -53,7 +53,8 @@ import { LocalSerializer } from './local_serializer';
 import {
   ActiveTargets,
   LruDelegate,
-  LruGarbageCollector
+  LruGarbageCollector,
+  LruParams
 } from './lru_garbage_collector';
 import { MutationQueue } from './mutation_queue';
 import {
@@ -193,14 +194,16 @@ export class IndexedDbPersistence implements Persistence {
     clientId: ClientId,
     platform: Platform,
     queue: AsyncQueue,
-    serializer: JsonProtoSerializer
+    serializer: JsonProtoSerializer,
+    lruParams: LruParams
   ): Promise<IndexedDbPersistence> {
     const persistence = new IndexedDbPersistence(
       persistenceKey,
       clientId,
       platform,
       queue,
-      serializer
+      serializer,
+      lruParams
     );
     await persistence.start();
     return persistence;
@@ -212,6 +215,7 @@ export class IndexedDbPersistence implements Persistence {
     platform: Platform,
     queue: AsyncQueue,
     serializer: JsonProtoSerializer,
+    lruParams: LruParams,
     multiClientParams: MultiClientParams
   ): Promise<IndexedDbPersistence> {
     const persistence = new IndexedDbPersistence(
@@ -220,6 +224,7 @@ export class IndexedDbPersistence implements Persistence {
       platform,
       queue,
       serializer,
+      lruParams,
       multiClientParams
     );
     await persistence.start();
@@ -271,6 +276,7 @@ export class IndexedDbPersistence implements Persistence {
     platform: Platform,
     private readonly queue: AsyncQueue,
     serializer: JsonProtoSerializer,
+    lruParams: LruParams,
     private readonly multiClientParams?: MultiClientParams
   ) {
     if (!IndexedDbPersistence.isAvailable()) {
@@ -279,7 +285,7 @@ export class IndexedDbPersistence implements Persistence {
         UNSUPPORTED_PLATFORM_ERROR_MSG
       );
     }
-    this.referenceDelegate = new IndexedDbLruDelegate(this);
+    this.referenceDelegate = new IndexedDbLruDelegate(this, lruParams);
     this.dbName = persistenceKey + IndexedDbPersistence.MAIN_DATABASE;
     this.serializer = new LocalSerializer(serializer);
     this.document = platform.document;
@@ -1071,8 +1077,8 @@ export class IndexedDbLruDelegate implements ReferenceDelegate, LruDelegate {
 
   readonly garbageCollector: LruGarbageCollector;
 
-  constructor(private readonly db: IndexedDbPersistence) {
-    this.garbageCollector = new LruGarbageCollector(this);
+  constructor(private readonly db: IndexedDbPersistence, params: LruParams) {
+    this.garbageCollector = new LruGarbageCollector(this, params);
   }
 
   getSequenceNumberCount(
@@ -1280,6 +1286,10 @@ export class IndexedDbLruDelegate implements ReferenceDelegate, LruDelegate {
         }
       });
   }
+
+  getCacheSize(txn: PersistenceTransaction): PersistencePromise<number> {
+    return this.db.getRemoteDocumentCache().getSize(txn);
+  }
 }
 
 function sentinelKey(key: DocumentKey): [TargetId, EncodedResourcePath] {