diff --git a/packages/firestore/src/local/indexeddb_schema.ts b/packages/firestore/src/local/indexeddb_schema.ts index 90904b6f717..0e01dfef9e7 100644 --- a/packages/firestore/src/local/indexeddb_schema.ts +++ b/packages/firestore/src/local/indexeddb_schema.ts @@ -31,7 +31,7 @@ import { DbTimestampKey } from './indexeddb_sentinels'; // TODO(indexing): Remove this constant const INDEXING_ENABLED = false; -export const INDEXING_SCHEMA_VERSION = 14; +export const INDEXING_SCHEMA_VERSION = 15; /** * Schema Version for the Web client: @@ -57,7 +57,7 @@ export const INDEXING_SCHEMA_VERSION = 14; * 14. Add indexing support. */ -export const SCHEMA_VERSION = INDEXING_ENABLED ? INDEXING_SCHEMA_VERSION : 13; +export const SCHEMA_VERSION = INDEXING_ENABLED ? INDEXING_SCHEMA_VERSION : 14; /** * Wrapper class to store timestamps (seconds and nanos) in IndexedDb objects. diff --git a/packages/firestore/src/local/indexeddb_schema_converter.ts b/packages/firestore/src/local/indexeddb_schema_converter.ts index d68fb613d7b..873060ab858 100644 --- a/packages/firestore/src/local/indexeddb_schema_converter.ts +++ b/packages/firestore/src/local/indexeddb_schema_converter.ts @@ -15,7 +15,10 @@ * limitations under the License. */ +import { User } from '../auth/user'; +import { ListenSequence } from '../core/listen_sequence'; import { SnapshotVersion } from '../core/snapshot_version'; +import { documentKeySet } from '../model/collections'; import { DocumentKey } from '../model/document_key'; import { ResourcePath } from '../model/path'; import { debugAssert, fail, hardAssert } from '../util/assert'; @@ -25,10 +28,13 @@ import { decodeResourcePath, encodeResourcePath } from './encoded_resource_path'; +import { IndexedDbDocumentOverlayCache } from './indexeddb_document_overlay_cache'; import { dbDocumentSize, removeMutationBatch } from './indexeddb_mutation_batch_impl'; +import { IndexedDbMutationQueue } from './indexeddb_mutation_queue'; +import { newIndexedDbRemoteDocumentCache } from './indexeddb_remote_document_cache'; import { DbCollectionParent, DbDocumentMutation, @@ -107,6 +113,8 @@ import { DbTargetQueryTargetsKeyPath, DbTargetStore } from './indexeddb_sentinels'; +import { IndexedDbTransaction } from './indexeddb_transaction'; +import { LocalDocumentsView } from './local_documents_view'; import { fromDbMutationBatch, fromDbTarget, @@ -114,6 +122,7 @@ import { toDbTarget } from './local_serializer'; import { MemoryCollectionParentIndex } from './memory_index_manager'; +import { MemoryEagerDelegate, MemoryPersistence } from './memory_persistence'; import { PersistencePromise } from './persistence_promise'; import { SimpleDbSchemaConverter, SimpleDbTransaction } from './simple_db'; @@ -240,9 +249,11 @@ export class SchemaConverter implements SimpleDbSchemaConverter { } if (fromVersion < 14 && toVersion >= 14) { - p = p.next(() => { - createFieldIndex(db); - }); + p = p.next(() => this.runOverlayMigration(db, simpleDbTransaction)); + } + + if (fromVersion < 15 && toVersion >= 15) { + p = p.next(() => createFieldIndex(db)); } return p; @@ -455,6 +466,97 @@ export class SchemaConverter implements SimpleDbSchemaConverter { }) .next(() => PersistencePromise.waitFor(writes)); } + + private runOverlayMigration( + db: IDBDatabase, + transaction: SimpleDbTransaction + ): PersistencePromise { + const queuesStore = transaction.store( + DbMutationQueueStore + ); + const mutationsStore = transaction.store< + DbMutationBatchKey, + DbMutationBatch + >(DbMutationBatchStore); + + const remoteDocumentCache = newIndexedDbRemoteDocumentCache( + this.serializer + ); + + const promises: Array> = []; + let userIds = new Set(); + + return queuesStore + .loadAll() + .next(queues => { + for (const queue of queues) { + const userId = queue.userId; + if (userIds.has(userId)) { + // We have already processed this user. + continue; + } + userIds = userIds.add(userId); + const user = new User(userId); + const documentOverlayCache = IndexedDbDocumentOverlayCache.forUser( + this.serializer, + user + ); + let allDocumentKeysForUser = documentKeySet(); + const range = IDBKeyRange.bound( + [userId, BATCHID_UNKNOWN], + [userId, Number.POSITIVE_INFINITY] + ); + promises.push( + mutationsStore + .loadAll(DbMutationBatchUserMutationsIndex, range) + .next(dbBatches => { + dbBatches.forEach(dbBatch => { + hardAssert( + dbBatch.userId === userId, + `Cannot process batch ${dbBatch.batchId} from unexpected user` + ); + const batch = fromDbMutationBatch(this.serializer, dbBatch); + batch + .keys() + .forEach( + key => + (allDocumentKeysForUser = + allDocumentKeysForUser.add(key)) + ); + }); + }) + .next(() => { + // NOTE: The index manager and the reference delegate are + // irrelevant for the purpose of recalculating and saving + // overlays. We can therefore simply use the memory + // implementation. + const memoryPersistence = new MemoryPersistence( + MemoryEagerDelegate.factory, + this.serializer.remoteSerializer + ); + const indexManager = memoryPersistence.getIndexManager(user); + const mutationQueue = IndexedDbMutationQueue.forUser( + user, + this.serializer, + indexManager, + memoryPersistence.referenceDelegate + ); + const localDocumentsView = new LocalDocumentsView( + remoteDocumentCache, + mutationQueue, + documentOverlayCache, + indexManager + ); + return localDocumentsView.recalculateAndSaveOverlaysForDocumentKeys( + new IndexedDbTransaction(transaction, ListenSequence.INVALID), + allDocumentKeysForUser + ); + }) + ); + } + }) + .next(() => PersistencePromise.waitFor(promises)); + } } function sentinelKey(path: ResourcePath): DbTargetDocumentKey { diff --git a/packages/firestore/src/local/indexeddb_sentinels.ts b/packages/firestore/src/local/indexeddb_sentinels.ts index 3a163ac9fe6..f0b92c1224a 100644 --- a/packages/firestore/src/local/indexeddb_sentinels.ts +++ b/packages/firestore/src/local/indexeddb_sentinels.ts @@ -407,8 +407,9 @@ export const V13_STORES = [ DbNamedQueryStore, DbDocumentOverlayStore ]; -export const V14_STORES = [ - ...V13_STORES, +export const V14_STORES = V13_STORES; +export const V15_STORES = [ + ...V14_STORES, DbIndexConfigurationStore, DbIndexStateStore, DbIndexEntryStore @@ -423,7 +424,9 @@ export const ALL_STORES = V12_STORES; /** Returns the object stores for the provided schema. */ export function getObjectStores(schemaVersion: number): string[] { - if (schemaVersion === 14) { + if (schemaVersion === 15) { + return V15_STORES; + } else if (schemaVersion === 14) { return V14_STORES; } else if (schemaVersion === 13) { return V13_STORES; diff --git a/packages/firestore/test/unit/local/indexeddb_persistence.test.ts b/packages/firestore/test/unit/local/indexeddb_persistence.test.ts index d3ff993801e..2528e8121ca 100644 --- a/packages/firestore/test/unit/local/indexeddb_persistence.test.ts +++ b/packages/firestore/test/unit/local/indexeddb_persistence.test.ts @@ -30,6 +30,7 @@ import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; import { DbCollectionParent, DbDocumentMutation, + DbDocumentOverlay, DbMutationBatch, DbMutationQueue, DbPrimaryClient, @@ -52,6 +53,8 @@ import { DbDocumentMutationKey, DbDocumentMutationPlaceholder, DbDocumentMutationStore, + DbDocumentOverlayKey, + DbDocumentOverlayStore, DbMutationBatchKey, DbMutationBatchStore, DbMutationQueueKey, @@ -81,6 +84,7 @@ import { import { fromDbTarget, LocalSerializer, + toDbDocumentOverlayKey, toDbRemoteDocument, toDbTarget, toDbTimestamp, @@ -92,6 +96,7 @@ import { ClientId } from '../../../src/local/shared_client_state'; import { SimpleDb, SimpleDbTransaction } from '../../../src/local/simple_db'; import { TargetData, TargetPurpose } from '../../../src/local/target_data'; import { MutableDocument } from '../../../src/model/document'; +import { DocumentKey } from '../../../src/model/document_key'; import { getWindow } from '../../../src/platform/dom'; import { firestoreV1ApiClientInterfaces } from '../../../src/protos/firestore_proto_api'; import { @@ -263,6 +268,23 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => { after(() => SimpleDb.delete(INDEXEDDB_TEST_DATABASE_NAME)); + function verifyUserHasDocumentOverlay( + txn: SimpleDbTransaction, + user: string, + doc: string, + expected: firestoreV1ApiClientInterfaces.Write + ): PersistencePromise { + const key = toDbDocumentOverlayKey(user, DocumentKey.fromPath(doc)); + const documentOverlayStore = txn.store< + DbDocumentOverlayKey, + DbDocumentOverlay + >(DbDocumentOverlayStore); + return documentOverlayStore.get(key).next(overlay => { + expect(overlay).to.not.be.null; + expect(overlay!.overlayMutation).to.deep.equal(expected); + }); + } + it('can install schema version 1', () => { return withDb(1, async (db, version, objectStores) => { expect(version).to.equal(1); @@ -1044,6 +1066,185 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => { }); }); + it('can upgrade from schema version 13 to 14 (overlay migration)', function (this: Context) { + // This test creates a database with schema version 13 that has three users, + // two of whom have local mutations. + const testWriteFoo = { + update: { + name: 'projects/test-project/databases/(default)/documents/docs/foo', + fields: {} + } + }; + const testWriteBar = { + update: { + name: 'projects/test-project/databases/(default)/documents/docs/bar', + fields: {} + } + }; + const testWriteBaz = { + update: { + name: 'projects/test-project/databases/(default)/documents/docs/baz', + fields: {} + } + }; + const testWriteNewDoc = { + update: { + name: 'projects/test-project/databases/(default)/documents/docs/newDoc', + fields: {} + } + }; + const testMutations: DbMutationBatch[] = [ + { + userId: 'user1', + batchId: 1, + localWriteTimeMs: 1337, + baseMutations: undefined, + mutations: [testWriteFoo] + }, + { + userId: 'user1', + batchId: 2, + localWriteTimeMs: 1337, + baseMutations: undefined, + mutations: [testWriteFoo] + }, + { + userId: 'user2', + batchId: 3, + localWriteTimeMs: 1337, + baseMutations: undefined, + mutations: [testWriteBar, testWriteBaz] + }, + { + userId: 'user2', + batchId: 4, + localWriteTimeMs: 1337, + baseMutations: undefined, + mutations: [testWriteNewDoc] + } + ]; + + return withDb(13, db => { + return db.runTransaction( + this.test!.fullTitle(), + 'readwrite', + V13_STORES, + txn => { + const mutationBatchStore = txn.store< + DbMutationBatchKey, + DbMutationBatch + >(DbMutationBatchStore); + const documentMutationStore = txn.store< + DbDocumentMutationKey, + DbDocumentMutation + >(DbDocumentMutationStore); + const mutationQueuesStore = txn.store< + DbMutationQueueKey, + DbMutationQueue + >(DbMutationQueueStore); + // Manually populate the mutation queue and create all indicies. + return PersistencePromise.forEach( + testMutations, + (testMutation: DbMutationBatch) => { + return mutationBatchStore.put(testMutation).next(() => { + return PersistencePromise.forEach( + testMutation.mutations, + (write: firestoreV1ApiClientInterfaces.Write) => { + const indexKey = newDbDocumentMutationKey( + testMutation.userId, + path(write.update!.name!, 5), + testMutation.batchId + ); + return documentMutationStore.put( + indexKey, + DbDocumentMutationPlaceholder + ); + } + ); + }); + } + ).next(() => + // Populate the mutation queues' metadata + PersistencePromise.waitFor([ + mutationQueuesStore.put({ + userId: 'user1', + lastAcknowledgedBatchId: -1, + lastStreamToken: '' + }), + mutationQueuesStore.put({ + userId: 'user2', + lastAcknowledgedBatchId: -1, + lastStreamToken: '' + }), + mutationQueuesStore.put({ + userId: 'user3', + lastAcknowledgedBatchId: -1, + lastStreamToken: '' + }) + ]) + ); + } + ); + }).then(() => + withDb(14, (db, version) => { + expect(version).to.be.equal(14); + + return db.runTransaction( + this.test!.fullTitle(), + 'readonly', + V14_STORES, + txn => { + const documentOverlayStore = txn.store< + DbDocumentOverlayKey, + DbDocumentOverlay + >(DbDocumentOverlayStore); + + // We should have a total of 4 overlays: + // For user1: testWriteFoo + // For user2: testWriteBar, testWriteBaz, and testWritePending + // For user3: NO OVERLAYS! + let p = documentOverlayStore.count().next(count => { + expect(count).to.equal(4); + }); + p = p.next(() => + verifyUserHasDocumentOverlay( + txn, + 'user1', + 'docs/foo', + testWriteFoo + ) + ); + p = p.next(() => + verifyUserHasDocumentOverlay( + txn, + 'user2', + 'docs/bar', + testWriteBar + ) + ); + p = p.next(() => + verifyUserHasDocumentOverlay( + txn, + 'user2', + 'docs/baz', + testWriteBaz + ) + ); + p = p.next(() => + verifyUserHasDocumentOverlay( + txn, + 'user2', + 'docs/newDoc', + testWriteNewDoc + ) + ); + return p; + } + ); + }) + ); + }); + it('can upgrade from version 13 to 14', async () => { await withDb(13, async () => {}); await withDb(14, async (db, version, objectStores) => {