From 2e28c3c5048ad970d8971441a03c4c6d726eea25 Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Thu, 8 Nov 2018 10:08:50 -0800 Subject: [PATCH 1/4] Implement sequence number migration --- .../firestore/src/local/indexeddb_schema.ts | 52 ++++++++++- .../unit/local/indexeddb_persistence.test.ts | 89 +++++++++++++++++++ 2 files changed, 140 insertions(+), 1 deletion(-) diff --git a/packages/firestore/src/local/indexeddb_schema.ts b/packages/firestore/src/local/indexeddb_schema.ts index 3a1f3500b56..512e0f250ed 100644 --- a/packages/firestore/src/local/indexeddb_schema.ts +++ b/packages/firestore/src/local/indexeddb_schema.ts @@ -23,6 +23,7 @@ import { SnapshotVersion } from '../core/snapshot_version'; import { BATCHID_UNKNOWN } from '../model/mutation_batch'; import { encode, EncodedResourcePath } from './encoded_resource_path'; import { removeMutationBatch } from './indexeddb_mutation_queue'; +import { getHighestListenSequenceNumber } from './indexeddb_query_cache'; import { dbDocumentSize } from './indexeddb_remote_document_cache'; import { LocalSerializer } from './local_serializer'; import { PersistencePromise } from './persistence_promise'; @@ -40,7 +41,7 @@ import { SimpleDbSchemaConverter, SimpleDbTransaction } from './simple_db'; * 4. Multi-Tab Support. * 5. Removal of held write acks. */ -export const SCHEMA_VERSION = 6; +export const SCHEMA_VERSION = 7; /** Performs database creation and schema upgrades. */ export class SchemaConverter implements SimpleDbSchemaConverter { @@ -115,6 +116,12 @@ export class SchemaConverter implements SimpleDbSchemaConverter { }); } + if (fromVersion < 7 && toVersion >= 7) { + p = p.next(() => { + return this.ensureSequenceNumbers(txn); + }); + } + return p; } @@ -172,6 +179,49 @@ export class SchemaConverter implements SimpleDbSchemaConverter { }); }); } + + /** + * Ensures that every document in the remote document cache has a corresponding sentinel row + * with a sequence number. Missing rows are given the most recently used sequence number. + */ + private ensureSequenceNumbers( + txn: SimpleDbTransaction + ): PersistencePromise { + const documentTargetStore = txn.store< + DbTargetDocumentKey, + DbTargetDocument + >(DbTargetDocument.store); + const documentsStore = txn.store( + DbRemoteDocument.store + ); + + return getHighestListenSequenceNumber(txn).next(currentSequenceNumber => { + const writeSentinelKey = ( + path: ResourcePath + ): PersistencePromise => { + return documentTargetStore.put( + new DbTargetDocument(0, encode(path), currentSequenceNumber) + ); + }; + + const promises: Array> = []; + return documentsStore + .iterate((key, doc) => { + const path = new ResourcePath(key); + const docSentinelKey = sentinelKey(path); + documentTargetStore.get(docSentinelKey).next(maybeSentinel => { + if (!maybeSentinel) { + promises.push(writeSentinelKey(path)); + } + }); + }) + .next(() => PersistencePromise.waitFor(promises)); + }); + } +} + +function sentinelKey(path: ResourcePath): DbTargetDocumentKey { + return [0, encode(path)]; } // TODO(mikelehen): Get rid of "as any" if/when TypeScript fixes their types. diff --git a/packages/firestore/test/unit/local/indexeddb_persistence.test.ts b/packages/firestore/test/unit/local/indexeddb_persistence.test.ts index 51bdb0baee6..bb289ad02a9 100644 --- a/packages/firestore/test/unit/local/indexeddb_persistence.test.ts +++ b/packages/firestore/test/unit/local/indexeddb_persistence.test.ts @@ -17,6 +17,7 @@ import { expect } from 'chai'; import { PersistenceSettings } from '../../../src/api/database'; import { SnapshotVersion } from '../../../src/core/snapshot_version'; +import { decode, encode } from '../../../src/local/encoded_resource_path'; import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; import { DbDocumentMutation, @@ -32,6 +33,8 @@ import { DbRemoteDocumentGlobalKey, DbRemoteDocumentKey, DbTarget, + DbTargetDocument, + DbTargetDocumentKey, DbTargetGlobal, DbTargetGlobalKey, DbTargetKey, @@ -521,6 +524,92 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => { }); }); }); + + it('can upgrade from version 6 to 7', async () => { + const oldSequenceNumber = 1; + // Set the highest sequence number to this value so that untagged documents + // will pick up this value. + const newSequenceNumber = 2; + await withDb(6, db => { + const serializer = TEST_SERIALIZER; + + const sdb = new SimpleDb(db); + return sdb.runTransaction('readwrite', V6_STORES, txn => { + const targetGlobalStore = txn.store( + DbTargetGlobal.store + ); + const remoteDocumentStore = txn.store< + DbRemoteDocumentKey, + DbRemoteDocument + >(DbRemoteDocument.store); + const targetDocumentStore = txn.store< + DbTargetDocumentKey, + DbTargetDocument + >(DbTargetDocument.store); + return targetGlobalStore + .get(DbTargetGlobal.key) + .next(metadata => { + expect(metadata).to.not.be.null; + metadata!.highestListenSequenceNumber = newSequenceNumber; + return targetGlobalStore.put(DbTargetGlobal.key, metadata!); + }) + .next(() => { + // Set up some documents (we only need the keys) + // For the odd ones, add sentinel rows. + const promises: Array> = []; + for (let i = 0; i < 10; i++) { + const document = doc('docs/doc_' + i, 1, { foo: 'bar' }); + promises.push( + remoteDocumentStore.put( + document.key.path.toArray(), + serializer.toDbRemoteDocument(document) + ) + ); + if (i % 2 === 1) { + promises.push( + targetDocumentStore.put( + new DbTargetDocument( + 0, + encode(document.key.path), + oldSequenceNumber + ) + ) + ); + } + } + return PersistencePromise.waitFor(promises); + }); + }); + }); + + // Now run the migration and verify + await withDb(7, db => { + const sdb = new SimpleDb(db); + return sdb.runTransaction('readonly', V6_STORES, txn => { + const targetDocumentStore = txn.store< + DbTargetDocumentKey, + DbTargetDocument + >(DbTargetDocument.store); + const range = IDBKeyRange.bound( + [0], + [1], + /*lowerOpen=*/ false, + /*upperOpen=*/ true + ); + return targetDocumentStore.iterate( + { range }, + ([_, path], targetDocument) => { + const decoded = decode(path); + const lastSegment = decoded.lastSegment(); + const docNum = +lastSegment.split('_')[1]; + const expected = + docNum % 2 === 1 ? oldSequenceNumber : newSequenceNumber; + expect(targetDocument.sequenceNumber).to.equal(expected); + } + ); + }); + }); + }); }); describe('IndexedDb: canActAsPrimary', () => { From 561d5b39e92d3efa3b405df6de67e87083fa93ef Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Fri, 9 Nov 2018 15:50:56 -0800 Subject: [PATCH 2/4] Document schema versions 6 and 7 --- packages/firestore/src/local/indexeddb_schema.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/firestore/src/local/indexeddb_schema.ts b/packages/firestore/src/local/indexeddb_schema.ts index 512e0f250ed..a9c0fc3d2dc 100644 --- a/packages/firestore/src/local/indexeddb_schema.ts +++ b/packages/firestore/src/local/indexeddb_schema.ts @@ -40,6 +40,8 @@ import { SimpleDbSchemaConverter, SimpleDbTransaction } from './simple_db'; * https://github.com/firebase/firebase-ios-sdk/issues/1548 * 4. Multi-Tab Support. * 5. Removal of held write acks. + * 6. Create document global for tracking document cache size. + * 7. Ensure every cached document has a sentinel row with a sequence number. */ export const SCHEMA_VERSION = 7; From 2ed1f6749da32710e8cdf244496edfff2e428ae1 Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Fri, 9 Nov 2018 15:52:05 -0800 Subject: [PATCH 3/4] Format to single line --- packages/firestore/src/local/indexeddb_schema.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/firestore/src/local/indexeddb_schema.ts b/packages/firestore/src/local/indexeddb_schema.ts index a9c0fc3d2dc..f87ae37fcc6 100644 --- a/packages/firestore/src/local/indexeddb_schema.ts +++ b/packages/firestore/src/local/indexeddb_schema.ts @@ -119,9 +119,7 @@ export class SchemaConverter implements SimpleDbSchemaConverter { } if (fromVersion < 7 && toVersion >= 7) { - p = p.next(() => { - return this.ensureSequenceNumbers(txn); - }); + p = p.next(() => this.ensureSequenceNumbers(txn)); } return p; From ea40ee4513992bc9ff857e82b2bfa9618ff01751 Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Fri, 9 Nov 2018 16:09:41 -0800 Subject: [PATCH 4/4] Ensure we wait for every read --- packages/firestore/src/local/indexeddb_schema.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/firestore/src/local/indexeddb_schema.ts b/packages/firestore/src/local/indexeddb_schema.ts index f87ae37fcc6..27f57d6de59 100644 --- a/packages/firestore/src/local/indexeddb_schema.ts +++ b/packages/firestore/src/local/indexeddb_schema.ts @@ -209,11 +209,15 @@ export class SchemaConverter implements SimpleDbSchemaConverter { .iterate((key, doc) => { const path = new ResourcePath(key); const docSentinelKey = sentinelKey(path); - documentTargetStore.get(docSentinelKey).next(maybeSentinel => { - if (!maybeSentinel) { - promises.push(writeSentinelKey(path)); - } - }); + promises.push( + documentTargetStore.get(docSentinelKey).next(maybeSentinel => { + if (!maybeSentinel) { + return writeSentinelKey(path); + } else { + return PersistencePromise.resolve(); + } + }) + ); }) .next(() => PersistencePromise.waitFor(promises)); });