From 0a979d0da23ecffc8c3d22faa3b844ff511cdf78 Mon Sep 17 00:00:00 2001 From: Ehsan Nasiri Date: Fri, 8 Apr 2022 13:14:57 -0500 Subject: [PATCH 1/4] Overlay migration --- .../src/local/indexeddb_schema_converter.ts | 104 +++++++++++++++++- 1 file changed, 101 insertions(+), 3 deletions(-) diff --git a/packages/firestore/src/local/indexeddb_schema_converter.ts b/packages/firestore/src/local/indexeddb_schema_converter.ts index d68fb613d7b..4fb79727785 100644 --- a/packages/firestore/src/local/indexeddb_schema_converter.ts +++ b/packages/firestore/src/local/indexeddb_schema_converter.ts @@ -15,7 +15,10 @@ * limitations under the License. */ +import { User } from '../auth/user'; +import { ListenSequence } from '../core/listen_sequence'; import { SnapshotVersion } from '../core/snapshot_version'; +import { documentKeySet } from '../model/collections'; import { DocumentKey } from '../model/document_key'; import { ResourcePath } from '../model/path'; import { debugAssert, fail, hardAssert } from '../util/assert'; @@ -25,10 +28,13 @@ import { decodeResourcePath, encodeResourcePath } from './encoded_resource_path'; +import { IndexedDbDocumentOverlayCache } from './indexeddb_document_overlay_cache'; import { dbDocumentSize, removeMutationBatch } from './indexeddb_mutation_batch_impl'; +import { IndexedDbMutationQueue } from './indexeddb_mutation_queue'; +import { newIndexedDbRemoteDocumentCache } from './indexeddb_remote_document_cache'; import { DbCollectionParent, DbDocumentMutation, @@ -107,6 +113,8 @@ import { DbTargetQueryTargetsKeyPath, DbTargetStore } from './indexeddb_sentinels'; +import { IndexedDbTransaction } from './indexeddb_transaction'; +import { LocalDocumentsView } from './local_documents_view'; import { fromDbMutationBatch, fromDbTarget, @@ -114,6 +122,7 @@ import { toDbTarget } from './local_serializer'; import { MemoryCollectionParentIndex } from './memory_index_manager'; +import { MemoryEagerDelegate, MemoryPersistence } from './memory_persistence'; import { PersistencePromise } from './persistence_promise'; import { SimpleDbSchemaConverter, SimpleDbTransaction } from './simple_db'; @@ -240,9 +249,9 @@ export class SchemaConverter implements SimpleDbSchemaConverter { } if (fromVersion < 14 && toVersion >= 14) { - p = p.next(() => { - createFieldIndex(db); - }); + p = p + .next(() => createFieldIndex(db)) + .next(() => this.runOverlayMigration(db, simpleDbTransaction)); } return p; @@ -455,6 +464,95 @@ export class SchemaConverter implements SimpleDbSchemaConverter { }) .next(() => PersistencePromise.waitFor(writes)); } + + private runOverlayMigration( + db: IDBDatabase, + transaction: SimpleDbTransaction + ): PersistencePromise { + const queuesStore = transaction.store( + DbMutationQueueStore + ); + const mutationsStore = transaction.store< + DbMutationBatchKey, + DbMutationBatch + >(DbMutationBatchStore); + + const promises: Array> = []; + let userIds = new Set(); + + return queuesStore + .loadAll() + .next(queues => { + for (const queue of queues) { + userIds = userIds.add(queue.userId); + } + }) + .next(() => { + userIds.forEach(userId => { + const user = new User(userId); + const remoteDocumentCache = newIndexedDbRemoteDocumentCache( + this.serializer + ); + const documentOverlayCache = IndexedDbDocumentOverlayCache.forUser( + this.serializer, + user + ); + let allDocumentKeysForUser = documentKeySet(); + const range = IDBKeyRange.bound( + [userId, BATCHID_UNKNOWN], + [userId, Number.POSITIVE_INFINITY] + ); + promises.push( + mutationsStore + .loadAll(DbMutationBatchUserMutationsIndex, range) + .next(dbBatches => { + dbBatches.forEach(dbBatch => { + hardAssert( + dbBatch.userId === userId, + `Cannot process batch ${dbBatch.batchId} from unexpected user` + ); + const batch = fromDbMutationBatch(this.serializer, dbBatch); + batch + .keys() + .forEach( + key => + (allDocumentKeysForUser = + allDocumentKeysForUser.add(key)) + ); + }); + }) + .next(() => { + // NOTE: The index manager and the reference delegate are + // irrelevant for the purpose of recalculating and saving + // overlays. We can therefore simply use the memory + // implementation. + const memoryPersistence = new MemoryPersistence( + MemoryEagerDelegate.factory, + this.serializer.remoteSerializer + ); + const indexManager = memoryPersistence.getIndexManager(user); + const mutationQueue = IndexedDbMutationQueue.forUser( + user, + this.serializer, + indexManager, + memoryPersistence.referenceDelegate + ); + const localDocumentsView = new LocalDocumentsView( + remoteDocumentCache, + mutationQueue, + documentOverlayCache, + indexManager + ); + return localDocumentsView.recalculateAndSaveOverlaysForDocumentKeys( + new IndexedDbTransaction(transaction, ListenSequence.INVALID), + allDocumentKeysForUser + ); + }) + ); + }); + }) + .next(() => PersistencePromise.waitFor(promises)); + } } function sentinelKey(path: ResourcePath): DbTargetDocumentKey { From 1adea546e2b1fc0f3eda113a0f6487981b6c1d15 Mon Sep 17 00:00:00 2001 From: Ehsan Nasiri Date: Wed, 13 Apr 2022 10:44:07 -0500 Subject: [PATCH 2/4] Remove intermediate state. --- .../src/local/indexeddb_schema_converter.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/firestore/src/local/indexeddb_schema_converter.ts b/packages/firestore/src/local/indexeddb_schema_converter.ts index 4fb79727785..572f712bc9a 100644 --- a/packages/firestore/src/local/indexeddb_schema_converter.ts +++ b/packages/firestore/src/local/indexeddb_schema_converter.ts @@ -484,11 +484,12 @@ export class SchemaConverter implements SimpleDbSchemaConverter { .loadAll() .next(queues => { for (const queue of queues) { - userIds = userIds.add(queue.userId); - } - }) - .next(() => { - userIds.forEach(userId => { + const userId = queue.userId; + if (userIds.has(userId)) { + // We have already processed this user. + continue; + } + userIds = userIds.add(userId); const user = new User(userId); const remoteDocumentCache = newIndexedDbRemoteDocumentCache( this.serializer @@ -549,7 +550,7 @@ export class SchemaConverter implements SimpleDbSchemaConverter { ); }) ); - }); + } }) .next(() => PersistencePromise.waitFor(promises)); } From 9b223570c506b9b7c8722185c70ba4479512efe3 Mon Sep 17 00:00:00 2001 From: Ehsan Nasiri Date: Thu, 14 Apr 2022 23:34:04 -0500 Subject: [PATCH 3/4] Add test. --- .../firestore/src/local/indexeddb_schema.ts | 4 +- .../src/local/indexeddb_schema_converter.ts | 15 +- .../src/local/indexeddb_sentinels.ts | 9 +- .../unit/local/indexeddb_persistence.test.ts | 213 ++++++++++++++++++ 4 files changed, 230 insertions(+), 11 deletions(-) diff --git a/packages/firestore/src/local/indexeddb_schema.ts b/packages/firestore/src/local/indexeddb_schema.ts index 90904b6f717..0e01dfef9e7 100644 --- a/packages/firestore/src/local/indexeddb_schema.ts +++ b/packages/firestore/src/local/indexeddb_schema.ts @@ -31,7 +31,7 @@ import { DbTimestampKey } from './indexeddb_sentinels'; // TODO(indexing): Remove this constant const INDEXING_ENABLED = false; -export const INDEXING_SCHEMA_VERSION = 14; +export const INDEXING_SCHEMA_VERSION = 15; /** * Schema Version for the Web client: @@ -57,7 +57,7 @@ export const INDEXING_SCHEMA_VERSION = 14; * 14. Add indexing support. */ -export const SCHEMA_VERSION = INDEXING_ENABLED ? INDEXING_SCHEMA_VERSION : 13; +export const SCHEMA_VERSION = INDEXING_ENABLED ? INDEXING_SCHEMA_VERSION : 14; /** * Wrapper class to store timestamps (seconds and nanos) in IndexedDb objects. diff --git a/packages/firestore/src/local/indexeddb_schema_converter.ts b/packages/firestore/src/local/indexeddb_schema_converter.ts index 572f712bc9a..873060ab858 100644 --- a/packages/firestore/src/local/indexeddb_schema_converter.ts +++ b/packages/firestore/src/local/indexeddb_schema_converter.ts @@ -249,9 +249,11 @@ export class SchemaConverter implements SimpleDbSchemaConverter { } if (fromVersion < 14 && toVersion >= 14) { - p = p - .next(() => createFieldIndex(db)) - .next(() => this.runOverlayMigration(db, simpleDbTransaction)); + p = p.next(() => this.runOverlayMigration(db, simpleDbTransaction)); + } + + if (fromVersion < 15 && toVersion >= 15) { + p = p.next(() => createFieldIndex(db)); } return p; @@ -477,6 +479,10 @@ export class SchemaConverter implements SimpleDbSchemaConverter { DbMutationBatch >(DbMutationBatchStore); + const remoteDocumentCache = newIndexedDbRemoteDocumentCache( + this.serializer + ); + const promises: Array> = []; let userIds = new Set(); @@ -491,9 +497,6 @@ export class SchemaConverter implements SimpleDbSchemaConverter { } userIds = userIds.add(userId); const user = new User(userId); - const remoteDocumentCache = newIndexedDbRemoteDocumentCache( - this.serializer - ); const documentOverlayCache = IndexedDbDocumentOverlayCache.forUser( this.serializer, user diff --git a/packages/firestore/src/local/indexeddb_sentinels.ts b/packages/firestore/src/local/indexeddb_sentinels.ts index 3a163ac9fe6..f0b92c1224a 100644 --- a/packages/firestore/src/local/indexeddb_sentinels.ts +++ b/packages/firestore/src/local/indexeddb_sentinels.ts @@ -407,8 +407,9 @@ export const V13_STORES = [ DbNamedQueryStore, DbDocumentOverlayStore ]; -export const V14_STORES = [ - ...V13_STORES, +export const V14_STORES = V13_STORES; +export const V15_STORES = [ + ...V14_STORES, DbIndexConfigurationStore, DbIndexStateStore, DbIndexEntryStore @@ -423,7 +424,9 @@ export const ALL_STORES = V12_STORES; /** Returns the object stores for the provided schema. */ export function getObjectStores(schemaVersion: number): string[] { - if (schemaVersion === 14) { + if (schemaVersion === 15) { + return V15_STORES; + } else if (schemaVersion === 14) { return V14_STORES; } else if (schemaVersion === 13) { return V13_STORES; diff --git a/packages/firestore/test/unit/local/indexeddb_persistence.test.ts b/packages/firestore/test/unit/local/indexeddb_persistence.test.ts index d3ff993801e..ec075c57812 100644 --- a/packages/firestore/test/unit/local/indexeddb_persistence.test.ts +++ b/packages/firestore/test/unit/local/indexeddb_persistence.test.ts @@ -30,6 +30,7 @@ import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; import { DbCollectionParent, DbDocumentMutation, + DbDocumentOverlay, DbMutationBatch, DbMutationQueue, DbPrimaryClient, @@ -52,6 +53,8 @@ import { DbDocumentMutationKey, DbDocumentMutationPlaceholder, DbDocumentMutationStore, + DbDocumentOverlayKey, + DbDocumentOverlayStore, DbMutationBatchKey, DbMutationBatchStore, DbMutationQueueKey, @@ -81,6 +84,7 @@ import { import { fromDbTarget, LocalSerializer, + toDbDocumentOverlayKey, toDbRemoteDocument, toDbTarget, toDbTimestamp, @@ -92,6 +96,7 @@ import { ClientId } from '../../../src/local/shared_client_state'; import { SimpleDb, SimpleDbTransaction } from '../../../src/local/simple_db'; import { TargetData, TargetPurpose } from '../../../src/local/target_data'; import { MutableDocument } from '../../../src/model/document'; +import { DocumentKey } from '../../../src/model/document_key'; import { getWindow } from '../../../src/platform/dom'; import { firestoreV1ApiClientInterfaces } from '../../../src/protos/firestore_proto_api'; import { @@ -1044,6 +1049,214 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => { }); }); + it('can upgrade from schema version 13 to 14 (overlay migration)', function (this: Context) { + // This test creates a database with schema version 13 that has three users, + // two of whom have local mutations. + const testWriteFoo = { + update: { + name: 'projects/test-project/databases/(default)/documents/docs/foo', + fields: {} + } + }; + const testWriteBar = { + update: { + name: 'projects/test-project/databases/(default)/documents/docs/bar', + fields: {} + } + }; + const testWriteBaz = { + update: { + name: 'projects/test-project/databases/(default)/documents/docs/baz', + fields: {} + } + }; + const testWritePending = { + update: { + name: 'projects/test-project/databases/(default)/documents/docs/pending', + fields: {} + } + }; + const testMutations: DbMutationBatch[] = [ + { + userId: 'user1', + batchId: 1, + localWriteTimeMs: 1337, + baseMutations: undefined, + mutations: [testWriteFoo] + }, + { + userId: 'user1', + batchId: 2, + localWriteTimeMs: 1337, + baseMutations: undefined, + mutations: [testWriteFoo] + }, + { + userId: 'user2', + batchId: 3, + localWriteTimeMs: 1337, + baseMutations: undefined, + mutations: [testWriteBar, testWriteBaz] + }, + { + userId: 'user2', + batchId: 4, + localWriteTimeMs: 1337, + baseMutations: undefined, + mutations: [testWritePending] + }, + { + userId: 'user1', + batchId: 5, + localWriteTimeMs: 1337, + baseMutations: undefined, + mutations: [testWritePending] + } + ]; + + return withDb(13, db => { + return db.runTransaction( + this.test!.fullTitle(), + 'readwrite', + V13_STORES, + txn => { + const mutationBatchStore = txn.store< + DbMutationBatchKey, + DbMutationBatch + >(DbMutationBatchStore); + const documentMutationStore = txn.store< + DbDocumentMutationKey, + DbDocumentMutation + >(DbDocumentMutationStore); + const mutationQueuesStore = txn.store< + DbMutationQueueKey, + DbMutationQueue + >(DbMutationQueueStore); + // Manually populate the mutation queue and create all indicies. + return PersistencePromise.forEach( + testMutations, + (testMutation: DbMutationBatch) => { + return mutationBatchStore.put(testMutation).next(() => { + return PersistencePromise.forEach( + testMutation.mutations, + (write: firestoreV1ApiClientInterfaces.Write) => { + const indexKey = newDbDocumentMutationKey( + testMutation.userId, + path(write.update!.name!, 5), + testMutation.batchId + ); + return documentMutationStore.put( + indexKey, + DbDocumentMutationPlaceholder + ); + } + ); + }); + } + ).next(() => + // Populate the mutation queues' metadata + PersistencePromise.waitFor([ + mutationQueuesStore.put({ + userId: 'user1', + lastAcknowledgedBatchId: -1, + lastStreamToken: '' + }), + mutationQueuesStore.put({ + userId: 'user2', + lastAcknowledgedBatchId: -1, + lastStreamToken: '' + }), + mutationQueuesStore.put({ + userId: 'user3', + lastAcknowledgedBatchId: -1, + lastStreamToken: '' + }) + ]) + ); + } + ); + }).then(() => + withDb(14, (db, version) => { + expect(version).to.be.equal(14); + + return db.runTransaction( + this.test!.fullTitle(), + 'readwrite', + V14_STORES, + txn => { + const documentOverlayStore = txn.store< + DbDocumentOverlayKey, + DbDocumentOverlay + >(DbDocumentOverlayStore); + + // We should have a total of 5 overlays: + // For user1: testWriteFoo, and testWritePending + // For user2: testWriteBar, testWriteBaz, and testWritePending + // For user3: NO OVERLAYS! + let p = documentOverlayStore.count().next(count => { + expect(count).to.equal(5); + }); + p = p.next(() => { + const key = toDbDocumentOverlayKey( + 'user1', + DocumentKey.fromPath('docs/foo') + ); + return documentOverlayStore.get(key).next(overlay => { + expect(overlay).to.not.be.null; + expect(overlay!.overlayMutation).to.deep.equal(testWriteFoo); + }); + }); + p = p.next(() => { + const key = toDbDocumentOverlayKey( + 'user1', + DocumentKey.fromPath('docs/pending') + ); + return documentOverlayStore.get(key).next(overlay => { + expect(overlay).to.not.be.null; + expect(overlay!.overlayMutation).to.deep.equal( + testWritePending + ); + }); + }); + p = p.next(() => { + const key = toDbDocumentOverlayKey( + 'user2', + DocumentKey.fromPath('docs/bar') + ); + return documentOverlayStore.get(key).next(overlay => { + expect(overlay).to.not.be.null; + expect(overlay!.overlayMutation).to.deep.equal(testWriteBar); + }); + }); + p = p.next(() => { + const key = toDbDocumentOverlayKey( + 'user2', + DocumentKey.fromPath('docs/baz') + ); + return documentOverlayStore.get(key).next(overlay => { + expect(overlay).to.not.be.null; + expect(overlay!.overlayMutation).to.deep.equal(testWriteBaz); + }); + }); + p = p.next(() => { + const key = toDbDocumentOverlayKey( + 'user2', + DocumentKey.fromPath('docs/pending') + ); + return documentOverlayStore.get(key).next(overlay => { + expect(overlay).to.not.be.null; + expect(overlay!.overlayMutation).to.deep.equal( + testWritePending + ); + }); + }); + return p; + } + ); + }) + ); + }); + it('can upgrade from version 13 to 14', async () => { await withDb(13, async () => {}); await withDb(14, async (db, version, objectStores) => { From 47d60573e16ad9bda3e79e6022ed471a9aab29a3 Mon Sep 17 00:00:00 2001 From: Ehsan Nasiri Date: Sun, 1 May 2022 22:17:48 -0500 Subject: [PATCH 4/4] Address feedback. --- .../unit/local/indexeddb_persistence.test.ts | 116 ++++++++---------- 1 file changed, 52 insertions(+), 64 deletions(-) diff --git a/packages/firestore/test/unit/local/indexeddb_persistence.test.ts b/packages/firestore/test/unit/local/indexeddb_persistence.test.ts index ec075c57812..2528e8121ca 100644 --- a/packages/firestore/test/unit/local/indexeddb_persistence.test.ts +++ b/packages/firestore/test/unit/local/indexeddb_persistence.test.ts @@ -268,6 +268,23 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => { after(() => SimpleDb.delete(INDEXEDDB_TEST_DATABASE_NAME)); + function verifyUserHasDocumentOverlay( + txn: SimpleDbTransaction, + user: string, + doc: string, + expected: firestoreV1ApiClientInterfaces.Write + ): PersistencePromise { + const key = toDbDocumentOverlayKey(user, DocumentKey.fromPath(doc)); + const documentOverlayStore = txn.store< + DbDocumentOverlayKey, + DbDocumentOverlay + >(DbDocumentOverlayStore); + return documentOverlayStore.get(key).next(overlay => { + expect(overlay).to.not.be.null; + expect(overlay!.overlayMutation).to.deep.equal(expected); + }); + } + it('can install schema version 1', () => { return withDb(1, async (db, version, objectStores) => { expect(version).to.equal(1); @@ -1070,9 +1087,9 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => { fields: {} } }; - const testWritePending = { + const testWriteNewDoc = { update: { - name: 'projects/test-project/databases/(default)/documents/docs/pending', + name: 'projects/test-project/databases/(default)/documents/docs/newDoc', fields: {} } }; @@ -1103,14 +1120,7 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => { batchId: 4, localWriteTimeMs: 1337, baseMutations: undefined, - mutations: [testWritePending] - }, - { - userId: 'user1', - batchId: 5, - localWriteTimeMs: 1337, - baseMutations: undefined, - mutations: [testWritePending] + mutations: [testWriteNewDoc] } ]; @@ -1181,7 +1191,7 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => { return db.runTransaction( this.test!.fullTitle(), - 'readwrite', + 'readonly', V14_STORES, txn => { const documentOverlayStore = txn.store< @@ -1189,67 +1199,45 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => { DbDocumentOverlay >(DbDocumentOverlayStore); - // We should have a total of 5 overlays: - // For user1: testWriteFoo, and testWritePending + // We should have a total of 4 overlays: + // For user1: testWriteFoo // For user2: testWriteBar, testWriteBaz, and testWritePending // For user3: NO OVERLAYS! let p = documentOverlayStore.count().next(count => { - expect(count).to.equal(5); - }); - p = p.next(() => { - const key = toDbDocumentOverlayKey( - 'user1', - DocumentKey.fromPath('docs/foo') - ); - return documentOverlayStore.get(key).next(overlay => { - expect(overlay).to.not.be.null; - expect(overlay!.overlayMutation).to.deep.equal(testWriteFoo); - }); + expect(count).to.equal(4); }); - p = p.next(() => { - const key = toDbDocumentOverlayKey( + p = p.next(() => + verifyUserHasDocumentOverlay( + txn, 'user1', - DocumentKey.fromPath('docs/pending') - ); - return documentOverlayStore.get(key).next(overlay => { - expect(overlay).to.not.be.null; - expect(overlay!.overlayMutation).to.deep.equal( - testWritePending - ); - }); - }); - p = p.next(() => { - const key = toDbDocumentOverlayKey( + 'docs/foo', + testWriteFoo + ) + ); + p = p.next(() => + verifyUserHasDocumentOverlay( + txn, 'user2', - DocumentKey.fromPath('docs/bar') - ); - return documentOverlayStore.get(key).next(overlay => { - expect(overlay).to.not.be.null; - expect(overlay!.overlayMutation).to.deep.equal(testWriteBar); - }); - }); - p = p.next(() => { - const key = toDbDocumentOverlayKey( + 'docs/bar', + testWriteBar + ) + ); + p = p.next(() => + verifyUserHasDocumentOverlay( + txn, 'user2', - DocumentKey.fromPath('docs/baz') - ); - return documentOverlayStore.get(key).next(overlay => { - expect(overlay).to.not.be.null; - expect(overlay!.overlayMutation).to.deep.equal(testWriteBaz); - }); - }); - p = p.next(() => { - const key = toDbDocumentOverlayKey( + 'docs/baz', + testWriteBaz + ) + ); + p = p.next(() => + verifyUserHasDocumentOverlay( + txn, 'user2', - DocumentKey.fromPath('docs/pending') - ); - return documentOverlayStore.get(key).next(overlay => { - expect(overlay).to.not.be.null; - expect(overlay!.overlayMutation).to.deep.equal( - testWritePending - ); - }); - }); + 'docs/newDoc', + testWriteNewDoc + ) + ); return p; } );