Overlay migration (#6131)

* Overlay migration

* Remove intermediate state.

* Add test.

* Address feedback.

Author: ehsannas

packages/firestore/src/local/indexeddb_schema.ts

@@ -31,7 +31,7 @@ import { DbTimestampKey } from './indexeddb_sentinels';
 // TODO(indexing): Remove this constant
 const INDEXING_ENABLED = false;
 
-export const INDEXING_SCHEMA_VERSION = 14;
+export const INDEXING_SCHEMA_VERSION = 15;
 
 /**
  * Schema Version for the Web client:
@@ -57,7 +57,7 @@ export const INDEXING_SCHEMA_VERSION = 14;
  * 14. Add indexing support.
  */
 
-export const SCHEMA_VERSION = INDEXING_ENABLED ? INDEXING_SCHEMA_VERSION : 13;
+export const SCHEMA_VERSION = INDEXING_ENABLED ? INDEXING_SCHEMA_VERSION : 14;
 
 /**
  * Wrapper class to store timestamps (seconds and nanos) in IndexedDb objects.

packages/firestore/src/local/indexeddb_schema_converter.ts

@@ -15,7 +15,10 @@
  * limitations under the License.
  */
 
+import { User } from '../auth/user';
+import { ListenSequence } from '../core/listen_sequence';
 import { SnapshotVersion } from '../core/snapshot_version';
+import { documentKeySet } from '../model/collections';
 import { DocumentKey } from '../model/document_key';
 import { ResourcePath } from '../model/path';
 import { debugAssert, fail, hardAssert } from '../util/assert';
@@ -25,10 +28,13 @@ import {
   decodeResourcePath,
   encodeResourcePath
 } from './encoded_resource_path';
+import { IndexedDbDocumentOverlayCache } from './indexeddb_document_overlay_cache';
 import {
   dbDocumentSize,
   removeMutationBatch
 } from './indexeddb_mutation_batch_impl';
+import { IndexedDbMutationQueue } from './indexeddb_mutation_queue';
+import { newIndexedDbRemoteDocumentCache } from './indexeddb_remote_document_cache';
 import {
   DbCollectionParent,
   DbDocumentMutation,
@@ -107,13 +113,16 @@ import {
   DbTargetQueryTargetsKeyPath,
   DbTargetStore
 } from './indexeddb_sentinels';
+import { IndexedDbTransaction } from './indexeddb_transaction';
+import { LocalDocumentsView } from './local_documents_view';
 import {
   fromDbMutationBatch,
   fromDbTarget,
   LocalSerializer,
   toDbTarget
 } from './local_serializer';
 import { MemoryCollectionParentIndex } from './memory_index_manager';
+import { MemoryEagerDelegate, MemoryPersistence } from './memory_persistence';
 import { PersistencePromise } from './persistence_promise';
 import { SimpleDbSchemaConverter, SimpleDbTransaction } from './simple_db';
 
@@ -240,9 +249,11 @@ export class SchemaConverter implements SimpleDbSchemaConverter {
     }
 
     if (fromVersion < 14 && toVersion >= 14) {
-      p = p.next(() => {
-        createFieldIndex(db);
-      });
+      p = p.next(() => this.runOverlayMigration(db, simpleDbTransaction));
+    }
+
+    if (fromVersion < 15 && toVersion >= 15) {
+      p = p.next(() => createFieldIndex(db));
     }
 
     return p;
@@ -455,6 +466,97 @@ export class SchemaConverter implements SimpleDbSchemaConverter {
       })
       .next(() => PersistencePromise.waitFor(writes));
   }
+
+  private runOverlayMigration(
+    db: IDBDatabase,
+    transaction: SimpleDbTransaction
+  ): PersistencePromise<void> {
+    const queuesStore = transaction.store<DbMutationQueueKey, DbMutationQueue>(
+      DbMutationQueueStore
+    );
+    const mutationsStore = transaction.store<
+      DbMutationBatchKey,
+      DbMutationBatch
+    >(DbMutationBatchStore);
+
+    const remoteDocumentCache = newIndexedDbRemoteDocumentCache(
+      this.serializer
+    );
+
+    const promises: Array<PersistencePromise<void>> = [];
+    let userIds = new Set<string>();
+
+    return queuesStore
+      .loadAll()
+      .next(queues => {
+        for (const queue of queues) {
+          const userId = queue.userId;
+          if (userIds.has(userId)) {
+            // We have already processed this user.
+            continue;
+          }
+          userIds = userIds.add(userId);
+          const user = new User(userId);
+          const documentOverlayCache = IndexedDbDocumentOverlayCache.forUser(
+            this.serializer,
+            user
+          );
+          let allDocumentKeysForUser = documentKeySet();
+          const range = IDBKeyRange.bound(
+            [userId, BATCHID_UNKNOWN],
+            [userId, Number.POSITIVE_INFINITY]
+          );
+          promises.push(
+            mutationsStore
+              .loadAll(DbMutationBatchUserMutationsIndex, range)
+              .next(dbBatches => {
+                dbBatches.forEach(dbBatch => {
+                  hardAssert(
+                    dbBatch.userId === userId,
+                    `Cannot process batch ${dbBatch.batchId} from unexpected user`
+                  );
+                  const batch = fromDbMutationBatch(this.serializer, dbBatch);
+                  batch
+                    .keys()
+                    .forEach(
+                      key =>
+                        (allDocumentKeysForUser =
+                          allDocumentKeysForUser.add(key))
+                    );
+                });
+              })
+              .next(() => {
+                // NOTE: The index manager and the reference delegate are
+                // irrelevant for the purpose of recalculating and saving
+                // overlays. We can therefore simply use the memory
+                // implementation.
+                const memoryPersistence = new MemoryPersistence(
+                  MemoryEagerDelegate.factory,
+                  this.serializer.remoteSerializer
+                );
+                const indexManager = memoryPersistence.getIndexManager(user);
+                const mutationQueue = IndexedDbMutationQueue.forUser(
+                  user,
+                  this.serializer,
+                  indexManager,
+                  memoryPersistence.referenceDelegate
+                );
+                const localDocumentsView = new LocalDocumentsView(
+                  remoteDocumentCache,
+                  mutationQueue,
+                  documentOverlayCache,
+                  indexManager
+                );
+                return localDocumentsView.recalculateAndSaveOverlaysForDocumentKeys(
+                  new IndexedDbTransaction(transaction, ListenSequence.INVALID),
+                  allDocumentKeysForUser
+                );
+              })
+          );
+        }
+      })
+      .next(() => PersistencePromise.waitFor(promises));
+  }
 }
 
 function sentinelKey(path: ResourcePath): DbTargetDocumentKey {

packages/firestore/src/local/indexeddb_sentinels.ts

@@ -407,8 +407,9 @@ export const V13_STORES = [
   DbNamedQueryStore,
   DbDocumentOverlayStore
 ];
-export const V14_STORES = [
-  ...V13_STORES,
+export const V14_STORES = V13_STORES;
+export const V15_STORES = [
+  ...V14_STORES,
   DbIndexConfigurationStore,
   DbIndexStateStore,
   DbIndexEntryStore
@@ -423,7 +424,9 @@ export const ALL_STORES = V12_STORES;
 
 /** Returns the object stores for the provided schema. */
 export function getObjectStores(schemaVersion: number): string[] {
-  if (schemaVersion === 14) {
+  if (schemaVersion === 15) {
+    return V15_STORES;
+  } else if (schemaVersion === 14) {
     return V14_STORES;
   } else if (schemaVersion === 13) {
     return V13_STORES;