diff --git a/packages/firebase/index.d.ts b/packages/firebase/index.d.ts index 2ecc282592b..7aed3e31a80 100644 --- a/packages/firebase/index.d.ts +++ b/packages/firebase/index.d.ts @@ -769,6 +769,27 @@ declare namespace firebase.firestore { timestampsInSnapshots?: boolean; } + // TODO(multitab): Uncomment when multi-tab is released publicly. + // /** + // * Settings that can be passed to Firestore.enablePersistence() to configure + // * Firestore persistence. + // */ + // export interface PersistenceSettings { + // /** + // * Whether to synchronize the in-memory state of multiple tabs. Setting this + // * to 'true' in all open tabs enables shared access to local persistence, + // * shared execution of queries and latency-compensated local document updates + // * across all connected instances. + // * + // * To enable this mode, `experimentalTabSynchronization:true` needs to be set + // * globally in all active tabs. If omitted or set to 'false', + // * `enablePersistence()` will fail in all but the first tab. + // * + // * NOTE: This mode is not yet recommended for production use. + // */ + // experimentalTabSynchronization?: boolean; + // } + export type LogLevel = 'debug' | 'error' | 'silent'; export function setLogLevel(logLevel: LogLevel): void; @@ -808,6 +829,29 @@ declare namespace firebase.firestore { */ enablePersistence(): Promise; + // TODO(multitab): Uncomment when multi-tab is released publicly. + // /** + // * Attempts to enable persistent storage, if possible. + // * + // * Must be called before any other methods (other than settings()). + // * + // * If this fails, enablePersistence() will reject the promise it returns. + // * Note that even after this failure, the firestore instance will remain + // * usable, however offline persistence will be disabled. + // * + // * There are several reasons why this can fail, which can be identified by + // * the `code` on the error. + // * + // * * failed-precondition: The app is already open in another browser tab. + // * * unimplemented: The browser is incompatible with the offline + // * persistence implementation. + // * + // * @param settings Optional settings object to configure persistence. + // * @return A promise that represents successfully enabling persistent + // * storage. + // */ + // enablePersistence(settings?: PersistenceSettings): Promise; + /** * Gets a `CollectionReference` instance that refers to the collection at * the specified path. diff --git a/packages/firestore-types/index.d.ts b/packages/firestore-types/index.d.ts index 18f348508a9..7e907f8488d 100644 --- a/packages/firestore-types/index.d.ts +++ b/packages/firestore-types/index.d.ts @@ -57,6 +57,27 @@ export interface Settings { timestampsInSnapshots?: boolean; } +// TODO(multitab): Uncomment when multi-tab is released publicly. +// /** +// * Settings that can be passed to Firestore.enablePersistence() to configure +// * Firestore persistence. +// */ +// export interface PersistenceSettings { +// /** +// * Whether to synchronize the in-memory state of multiple tabs. Setting this +// * to 'true' in all open tabs enables shared access to local persistence, +// * shared execution of queries and latency-compensated local document updates +// * across all connected instances. +// * +// * To enable this mode, `experimentalTabSynchronization:true` needs to be set +// * globally in all active tabs. If omitted or set to 'false', +// * `enablePersistence()` will fail in all but the first tab. +// * +// * NOTE: This mode is not yet recommended for production use. +// */ +// experimentalTabSynchronization?: boolean; +// } + export type LogLevel = 'debug' | 'error' | 'silent'; export function setLogLevel(logLevel: LogLevel): void; @@ -96,6 +117,29 @@ export class FirebaseFirestore { */ enablePersistence(): Promise; + // TODO(multitab): Uncomment when multi-tab is released publicly. + // /** + // * Attempts to enable persistent storage, if possible. + // * + // * Must be called before any other methods (other than settings()). + // * + // * If this fails, enablePersistence() will reject the promise it returns. + // * Note that even after this failure, the firestore instance will remain + // * usable, however offline persistence will be disabled. + // * + // * There are several reasons why this can fail, which can be identified by + // * the `code` on the error. + // * + // * * failed-precondition: The app is already open in another browser tab. + // * * unimplemented: The browser is incompatible with the offline + // * persistence implementation. + // * + // * @param settings Optional settings object to configure persistence. + // * @return A promise that represents successfully enabling persistent + // * storage. + // */ + // enablePersistence(settings?: PersistenceSettings): Promise; + /** * Gets a `CollectionReference` instance that refers to the collection at * the specified path. diff --git a/packages/firestore/.idea/runConfigurations/All_Tests.xml b/packages/firestore/.idea/runConfigurations/All_Tests.xml index cec05d4e5ba..eeb610c275d 100644 --- a/packages/firestore/.idea/runConfigurations/All_Tests.xml +++ b/packages/firestore/.idea/runConfigurations/All_Tests.xml @@ -4,7 +4,7 @@ $PROJECT_DIR$ true - + bdd --require ts-node/register/type-check --require index.node.ts --timeout 5000 PATTERN diff --git a/packages/firestore/.idea/runConfigurations/Integration_Tests.xml b/packages/firestore/.idea/runConfigurations/Integration_Tests.xml index 42d152fa293..aa5ed0d8f9f 100644 --- a/packages/firestore/.idea/runConfigurations/Integration_Tests.xml +++ b/packages/firestore/.idea/runConfigurations/Integration_Tests.xml @@ -4,7 +4,7 @@ $PROJECT_DIR$ true - + bdd --require ts-node/register/type-check --require index.node.ts --timeout 5000 PATTERN diff --git a/packages/firestore/.idea/runConfigurations/Unit_Tests.xml b/packages/firestore/.idea/runConfigurations/Unit_Tests.xml index 9230817893c..24ccb285c5d 100644 --- a/packages/firestore/.idea/runConfigurations/Unit_Tests.xml +++ b/packages/firestore/.idea/runConfigurations/Unit_Tests.xml @@ -4,7 +4,7 @@ $PROJECT_DIR$ true - + bdd --require ts-node/register/type-check --require index.node.ts PATTERN diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index 9d71d402c0f..e0af8bfe9c1 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -97,10 +97,14 @@ import { // underscore to discourage their use. // tslint:disable:strip-private-property-underscore +// settings() defaults: const DEFAULT_HOST = 'firestore.googleapis.com'; const DEFAULT_SSL = true; const DEFAULT_TIMESTAMPS_IN_SNAPSHOTS = false; +// enablePersistence() defaults: +const DEFAULT_SYNCHRONIZE_TABS = false; + /** Undocumented, private additional settings not exposed in our public API. */ interface PrivateSettings extends firestore.Settings { // Can be a google-auth-library or gapi client. @@ -197,6 +201,39 @@ class FirestoreConfig { persistence: boolean; } +// TODO(multitab): Replace with Firestore.PersistenceSettings +// tslint:disable-next-line:no-any The definition for these settings is private +export type _PersistenceSettings = any; + +/** + * Encapsulates the settings that can be used to configure Firestore + * persistence. + */ +export class PersistenceSettings { + /** Whether to enable multi-tab synchronization. */ + experimentalTabSynchronization: boolean; + + constructor(readonly enabled: boolean, settings?: _PersistenceSettings) { + assert( + enabled || !settings, + 'Can only provide PersistenceSettings with persistence enabled' + ); + settings = settings || {}; + this.experimentalTabSynchronization = objUtils.defaulted( + settings.experimentalTabSynchronization, + DEFAULT_SYNCHRONIZE_TABS + ); + } + + isEqual(other: PersistenceSettings): boolean { + return ( + this.enabled === other.enabled && + this.experimentalTabSynchronization === + other.experimentalTabSynchronization + ); + } +} + /** * The root reference to the database. */ @@ -290,7 +327,7 @@ export class Firestore implements firestore.FirebaseFirestore, FirebaseService { return this._firestoreClient.disableNetwork(); } - enablePersistence(): Promise { + enablePersistence(settings?: _PersistenceSettings): Promise { if (this._firestoreClient) { throw new FirestoreError( Code.FAILED_PRECONDITION, @@ -300,19 +337,23 @@ export class Firestore implements firestore.FirebaseFirestore, FirebaseService { ); } - return this.configureClient(/* persistence= */ true); + return this.configureClient( + new PersistenceSettings(/* enabled= */ true, settings) + ); } ensureClientConfigured(): FirestoreClient { if (!this._firestoreClient) { // Kick off starting the client but don't actually wait for it. // tslint:disable-next-line:no-floating-promises - this.configureClient(/* persistence= */ false); + this.configureClient(new PersistenceSettings(/* enabled= */ false)); } return this._firestoreClient as FirestoreClient; } - private configureClient(persistence: boolean): Promise { + private configureClient( + persistenceSettings: PersistenceSettings + ): Promise { assert( !!this._config.settings.host, 'FirestoreSettings.host cannot be falsey' @@ -378,7 +419,7 @@ follow these steps, YOUR APP MAY BREAK.`); this._config.credentials, this._queue ); - return this._firestoreClient.start(persistence); + return this._firestoreClient.start(persistenceSettings); } private static databaseIdFromApp(app: FirebaseApp): DatabaseId { diff --git a/packages/firestore/src/core/event_manager.ts b/packages/firestore/src/core/event_manager.ts index b8104d6186f..b31006437a2 100644 --- a/packages/firestore/src/core/event_manager.ts +++ b/packages/firestore/src/core/event_manager.ts @@ -15,11 +15,10 @@ */ import { Query } from './query'; -import { SyncEngine } from './sync_engine'; +import { SyncEngine, SyncEngineListener } from './sync_engine'; import { OnlineState, TargetId } from './types'; import { DocumentViewChange } from './view_snapshot'; import { ChangeType, ViewSnapshot } from './view_snapshot'; -import { DocumentSet } from '../model/document_set'; import { assert } from '../util/assert'; import { EventHandler } from '../util/misc'; import { ObjectMap } from '../util/obj_map'; @@ -47,7 +46,7 @@ export interface Observer { * It handles "fan-out". -- Identical queries will re-use the same watch on the * backend. */ -export class EventManager { +export class EventManager implements SyncEngineListener { private queries = new ObjectMap(q => q.canonicalId() ); @@ -55,10 +54,7 @@ export class EventManager { private onlineState: OnlineState = OnlineState.Unknown; constructor(private syncEngine: SyncEngine) { - this.syncEngine.subscribe( - this.onChange.bind(this), - this.onError.bind(this) - ); + this.syncEngine.subscribe(this); } listen(listener: QueryListener): Promise { @@ -106,7 +102,7 @@ export class EventManager { } } - onChange(viewSnaps: ViewSnapshot[]): void { + onWatchChange(viewSnaps: ViewSnapshot[]): void { for (const viewSnap of viewSnaps) { const query = viewSnap.query; const queryInfo = this.queries.get(query); @@ -119,7 +115,7 @@ export class EventManager { } } - onError(query: Query, error: Error): void { + onWatchError(query: Query, error: Error): void { const queryInfo = this.queries.get(query); if (queryInfo) { for (const listener of queryInfo.listeners) { @@ -132,7 +128,7 @@ export class EventManager { this.queries.delete(query); } - applyOnlineStateChange(onlineState: OnlineState): void { + onOnlineStateChange(onlineState: OnlineState): void { this.onlineState = onlineState; this.queries.forEach((_, queryInfo) => { for (const listener of queryInfo.listeners) { @@ -289,28 +285,13 @@ export class QueryListener { !this.raisedInitialEvent, 'Trying to raise initial events for second time' ); - snap = new ViewSnapshot( + snap = ViewSnapshot.fromInitialDocuments( snap.query, snap.docs, - DocumentSet.emptySet(snap.docs), - QueryListener.getInitialViewChanges(snap), snap.fromCache, - snap.hasPendingWrites, - /* syncChangesState= */ true, - /* excludesMetadataChanges= */ false + snap.hasPendingWrites ); this.raisedInitialEvent = true; this.queryObserver.next(snap); } - - /** Returns changes as if all documents in the snap were added. */ - private static getInitialViewChanges( - snap: ViewSnapshot - ): DocumentViewChange[] { - const result: DocumentViewChange[] = []; - snap.docs.forEach(doc => { - result.push({ type: ChangeType.Added, doc }); - }); - return result; - } } diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index 6a6a6cb330e..e75dc7f6bb2 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -51,8 +51,16 @@ import { Deferred } from '../util/promise'; import { DatabaseId, DatabaseInfo } from './database_info'; import { Query } from './query'; import { Transaction } from './transaction'; -import { OnlineState } from './types'; +import { OnlineStateSource } from './types'; import { ViewSnapshot } from './view_snapshot'; +import { + MemorySharedClientState, + SharedClientState, + WebStorageSharedClientState +} from '../local/shared_client_state'; +import { AutoId } from '../util/misc'; +import { PersistenceSettings } from '../api/database'; +import { assert } from '../util/assert'; const LOG_TAG = 'FirestoreClient'; @@ -81,6 +89,11 @@ export class FirestoreClient { private remoteStore: RemoteStore; private syncEngine: SyncEngine; + private readonly clientId = AutoId.newId(); + + // PORTING NOTE: SharedClientState is only used for multi-tab web. + private sharedClientState: SharedClientState; + constructor( private platform: Platform, private databaseInfo: DatabaseInfo, @@ -124,13 +137,14 @@ export class FirestoreClient { * fallback succeeds we signal success to the async queue even though the * start() itself signals failure. * - * @param usePersistence Whether or not to attempt to enable persistence. + * @param persistenceSettings Settings object to configure offline + * persistence. * @returns A deferred result indicating the user-visible result of enabling * offline persistence. This method will reject this if IndexedDB fails to * start for any reason. If usePersistence is false this is * unconditionally resolved. */ - start(usePersistence: boolean): Promise { + start(persistenceSettings: PersistenceSettings): Promise { // We defer our initialization until we get the current user from // setUserChangeListener(). We block the async queue until we got the // initial user and the initialization is completed. This will prevent @@ -153,7 +167,7 @@ export class FirestoreClient { if (!initialized) { initialized = true; - this.initializePersistence(usePersistence, persistenceResult) + this.initializePersistence(persistenceSettings, persistenceResult, user) .then(() => this.initializeRest(user)) .then(initializationDone.resolve, initializationDone.reject); } else { @@ -177,7 +191,7 @@ export class FirestoreClient { /** Enables the network connection and requeues all pending operations. */ enableNetwork(): Promise { return this.asyncQueue.enqueue(() => { - return this.remoteStore.enableNetwork(); + return this.syncEngine.enableNetwork(); }); } @@ -189,7 +203,7 @@ export class FirestoreClient { * platform can't possibly support our implementation then this method rejects * the persistenceResult and falls back on memory-only persistence. * - * @param usePersistence indicates whether or not to use offline persistence + * @param persistenceSettings Settings object to configure offline persistence * @param persistenceResult A deferred result indicating the user-visible * result of enabling offline persistence. This method will reject this if * IndexedDB fails to start for any reason. If usePersistence is false @@ -199,11 +213,12 @@ export class FirestoreClient { * succeeded. */ private initializePersistence( - usePersistence: boolean, - persistenceResult: Deferred + persistenceSettings: PersistenceSettings, + persistenceResult: Deferred, + user: User ): Promise { - if (usePersistence) { - return this.startIndexedDbPersistence() + if (persistenceSettings.enabled) { + return this.startIndexedDbPersistence(user, persistenceSettings) .then(persistenceResult.resolve) .catch(error => { // Regardless of whether or not the retry succeeds, from an user @@ -266,7 +281,15 @@ export class FirestoreClient { * * @returns A promise indicating success or failure. */ - private startIndexedDbPersistence(): Promise { + private startIndexedDbPersistence( + user: User, + settings: PersistenceSettings + ): Promise { + assert( + settings.enabled, + 'Should only start IndexedDb persitence with offline persistence enabled.' + ); + // TODO(http://b/33384523): For now we just disable garbage collection // when persistence is enabled. this.garbageCollector = new NoOpGarbageCollector(); @@ -277,8 +300,38 @@ export class FirestoreClient { const serializer = new JsonProtoSerializer(this.databaseInfo.databaseId, { useProto3Json: true }); - this.persistence = new IndexedDbPersistence(storagePrefix, serializer); - return this.persistence.start(); + + return Promise.resolve().then(() => { + const persistence: IndexedDbPersistence = new IndexedDbPersistence( + storagePrefix, + this.clientId, + this.platform, + this.asyncQueue, + serializer + ); + this.persistence = persistence; + + if ( + settings.experimentalTabSynchronization && + !WebStorageSharedClientState.isAvailable(this.platform) + ) { + throw new FirestoreError( + Code.UNIMPLEMENTED, + 'IndexedDB persistence is only available on platforms that support LocalStorage.' + ); + } + + this.sharedClientState = settings.experimentalTabSynchronization + ? new WebStorageSharedClientState( + this.asyncQueue, + this.platform, + storagePrefix, + this.clientId, + user + ) + : new MemorySharedClientState(); + return persistence.start(settings.experimentalTabSynchronization); + }); } /** @@ -288,7 +341,8 @@ export class FirestoreClient { */ private startMemoryPersistence(): Promise { this.garbageCollector = new EagerGarbageCollector(); - this.persistence = new MemoryPersistence(); + this.persistence = new MemoryPersistence(this.clientId); + this.sharedClientState = new MemorySharedClientState(); return this.persistence.start(); } @@ -300,11 +354,12 @@ export class FirestoreClient { private initializeRest(user: User): Promise { return this.platform .loadConnection(this.databaseInfo) - .then(connection => { + .then(async connection => { this.localStore = new LocalStore( this.persistence, user, - this.garbageCollector + this.garbageCollector, + this.sharedClientState ); const serializer = this.platform.newSerializer( this.databaseInfo.databaseId @@ -316,36 +371,51 @@ export class FirestoreClient { serializer ); - const onlineStateChangedHandler = (onlineState: OnlineState) => { - this.syncEngine.applyOnlineStateChange(onlineState); - this.eventMgr.applyOnlineStateChange(onlineState); - }; + const remoteStoreOnlineStateChangedHandler = onlineState => + this.syncEngine.applyOnlineStateChange( + onlineState, + OnlineStateSource.RemoteStore + ); + const sharedClientStateOnlineStateChangedHandler = onlineState => + this.syncEngine.applyOnlineStateChange( + onlineState, + OnlineStateSource.SharedClientState + ); this.remoteStore = new RemoteStore( this.localStore, datastore, this.asyncQueue, - onlineStateChangedHandler + remoteStoreOnlineStateChangedHandler ); this.syncEngine = new SyncEngine( this.localStore, this.remoteStore, + this.sharedClientState, user ); - // Setup wiring between sync engine and remote store + this.sharedClientState.onlineStateHandler = sharedClientStateOnlineStateChangedHandler; + + // Set up wiring between sync engine and other components this.remoteStore.syncEngine = this.syncEngine; + this.sharedClientState.syncEngine = this.syncEngine; this.eventMgr = new EventManager(this.syncEngine); - // NOTE: RemoteStore depends on LocalStore (for persisting stream - // tokens, refilling mutation queue, etc.) so must be started after - // LocalStore. - return this.localStore.start(); - }) - .then(() => { - return this.remoteStore.start(); + // NOTE: SyncEngine depends on both LocalStore and SharedClientState + // (for persisting stream tokens, refilling mutation queue, retrieving + // the list of active targets, etc.) so it must be started last. + await this.localStore.start(); + await this.sharedClientState.start(); + await this.remoteStore.start(); + + // NOTE: This will immediately call the listener, so we make sure to + // set it after localStore / remoteStore are started. + await this.persistence.setPrimaryStateListener(isPrimary => + this.syncEngine.applyPrimaryState(isPrimary) + ); }); } @@ -359,24 +429,26 @@ export class FirestoreClient { /** Disables the network connection. Pending operations will not complete. */ disableNetwork(): Promise { return this.asyncQueue.enqueue(() => { - return this.remoteStore.disableNetwork(); + return this.syncEngine.disableNetwork(); }); } shutdown(options?: { purgePersistenceWithDataLoss?: boolean; }): Promise { - return this.asyncQueue - .enqueue(() => { - this.credentials.removeUserChangeListener(); - return this.remoteStore.shutdown(); - }) - .then(() => { - // PORTING NOTE: LocalStore does not need an explicit shutdown on web. - return this.persistence.shutdown( - options && options.purgePersistenceWithDataLoss - ); - }); + return this.asyncQueue.enqueue(async () => { + // PORTING NOTE: LocalStore does not need an explicit shutdown on web. + await this.remoteStore.shutdown(); + await this.sharedClientState.shutdown(); + await this.persistence.shutdown( + options && options.purgePersistenceWithDataLoss + ); + + // `removeUserChangeListener` must be called after shutting down the + // RemoteStore as it will prevent the RemoteStore from retrieving + // auth tokens. + this.credentials.removeUserChangeListener(); + }); } listen( @@ -429,7 +501,10 @@ export class FirestoreClient { const viewDocChanges: ViewDocumentChanges = view.computeDocChanges( docs ); - return view.applyChanges(viewDocChanges).snapshot; + return view.applyChanges( + viewDocChanges, + /* updateLimboDocuments= */ false + ).snapshot; }); } diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 30f92ff5bea..6319a2e85f5 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -29,14 +29,13 @@ import { MaybeDocument, NoDocument } from '../model/document'; import { DocumentKey } from '../model/document_key'; import { Mutation } from '../model/mutation'; import { MutationBatchResult } from '../model/mutation_batch'; -import { RemoteEvent } from '../remote/remote_event'; +import { RemoteEvent, TargetChange } from '../remote/remote_event'; import { RemoteStore } from '../remote/remote_store'; import { RemoteSyncer } from '../remote/remote_syncer'; import { assert, fail } from '../util/assert'; import { FirestoreError } from '../util/error'; import * as log from '../util/log'; import { AnyJs, primitiveComparator } from '../util/misc'; -import * as objUtils from '../util/obj'; import { ObjectMap } from '../util/obj_map'; import { Deferred } from '../util/promise'; import { SortedMap } from '../util/sorted_map'; @@ -46,22 +45,33 @@ import { Query } from './query'; import { SnapshotVersion } from './snapshot_version'; import { TargetIdGenerator } from './target_id_generator'; import { Transaction } from './transaction'; -import { BatchId, OnlineState, TargetId } from './types'; +import { + BatchId, + MutationBatchState, + OnlineState, + OnlineStateSource, + TargetId +} from './types'; import { AddedLimboDocument, LimboDocumentChange, RemovedLimboDocument, View, + ViewChange, ViewDocumentChanges } from './view'; import { ViewSnapshot } from './view_snapshot'; +import { + SharedClientStateSyncer, + QueryTargetState +} from '../local/shared_client_state_syncer'; +import { ClientId, SharedClientState } from '../local/shared_client_state'; import { SortedSet } from '../util/sorted_set'; +import * as objUtils from '../util/obj'; +import { isPrimaryLeaseLostError } from '../local/indexeddb_persistence'; const LOG_TAG = 'SyncEngine'; -export type ViewHandler = (viewSnaps: ViewSnapshot[]) => void; -export type ErrorHandler = (query: Query, error: Error) => void; - /** * QueryView contains all of the data that SyncEngine needs to keep track of for * a particular query. @@ -100,6 +110,21 @@ class LimboResolution { receivedDocument: boolean; } +/** + * Interface implemented by EventManager to handle notifications from + * SyncEngine. + */ +export interface SyncEngineListener { + /** Handles new view snapshots. */ + onWatchChange(snapshots: ViewSnapshot[]): void; + + /** Handles the failure of a query. */ + onWatchError(query: Query, error: Error): void; + + /** Handles a change in online state. */ + onOnlineStateChange(onlineState: OnlineState): void; +} + /** * SyncEngine is the central controller in the client SDK architecture. It is * the glue code between the EventManager, LocalStore, and RemoteStore. Some of @@ -114,9 +139,8 @@ class LimboResolution { * The SyncEngine’s methods should only ever be called by methods running in the * global async queue. */ -export class SyncEngine implements RemoteSyncer { - private viewHandler: ViewHandler | null = null; - private errorHandler: ErrorHandler | null = null; +export class SyncEngine implements RemoteSyncer, SharedClientStateSyncer { + private syncEngineListener: SyncEngineListener | null = null; private queryViewsByQuery = new ObjectMap(q => q.canonicalId() @@ -134,28 +158,38 @@ export class SyncEngine implements RemoteSyncer { private mutationUserCallbacks = {} as { [uidKey: string]: SortedMap>; }; - private targetIdGenerator = TargetIdGenerator.forSyncEngine(); + private limboTargetIdGenerator = TargetIdGenerator.forSyncEngine(); + + // The primary state is set to `true` or `false` immediately after Firestore + // startup. In the interim, a client should only be considered primary if + // `isPrimary` is true. + private isPrimary: undefined | boolean = undefined; + private onlineState: OnlineState = OnlineState.Unknown; constructor( private localStore: LocalStore, private remoteStore: RemoteStore, + // PORTING NOTE: Manages state synchronization in multi-tab environments. + private sharedClientState: SharedClientState, private currentUser: User ) { this.limboCollector.addGarbageSource(this.limboDocumentRefs); } - /** Subscribes view and error handler. Can be called only once. */ - subscribe(viewHandler: ViewHandler, errorHandler: ErrorHandler): void { - assert( - viewHandler !== null && errorHandler !== null, - 'View and error handlers cannot be null' - ); + // Only used for testing. + get isPrimaryClient(): boolean { + return this.isPrimary === true; + } + + /** Subscribes to SyncEngine notifications. Has to be called exactly once. */ + subscribe(syncEngineListener: SyncEngineListener): void { + assert(syncEngineListener !== null, 'SyncEngine listener cannot be null'); assert( - this.viewHandler === null && this.errorHandler === null, + this.syncEngineListener === null, 'SyncEngine already has a subscriber.' ); - this.viewHandler = viewHandler; - this.errorHandler = errorHandler; + + this.syncEngineListener = syncEngineListener; } /** @@ -163,58 +197,141 @@ export class SyncEngine implements RemoteSyncer { * server. All the subsequent view snapshots or errors are sent to the * subscribed handlers. Returns the targetId of the query. */ - listen(query: Query): Promise { + async listen(query: Query): Promise { this.assertSubscribed('listen()'); - assert( - !this.queryViewsByQuery.has(query), - 'We already listen to the query: ' + query - ); - return this.localStore.allocateQuery(query).then(queryData => { + let targetId; + let viewSnapshot; + + const queryView = this.queryViewsByQuery.get(query); + if (queryView) { + // PORTING NOTE: With Mult-Tab Web, it is possible that a query view + // already exists when EventManager calls us for the first time. This + // happens when the primary tab is already listening to this query on + // behalf of another tab and the user of the primary also starts listening + // to the query. EventManager will not have an assigned target ID in this + // case and calls `listen` to obtain this ID. + targetId = queryView.targetId; + this.sharedClientState.addLocalQueryTarget(targetId); + viewSnapshot = queryView.view.computeInitialSnapshot(); + } else { + const queryData = await this.localStore.allocateQuery(query); + const status = this.sharedClientState.addLocalQueryTarget( + queryData.targetId + ); + targetId = queryData.targetId; + viewSnapshot = await this.initializeViewAndComputeSnapshot( + queryData, + status === 'current' + ); + if (this.isPrimary) { + this.remoteStore.listen(queryData); + } + } + + this.syncEngineListener!.onWatchChange([viewSnapshot]); + return targetId; + } + + /** + * Registers a view for a previously unknown query and computes its initial + * snapshot. + */ + private initializeViewAndComputeSnapshot( + queryData: QueryData, + current: boolean + ): Promise { + const query = queryData.query; + + return this.localStore.executeQuery(query).then(docs => { return this.localStore - .executeQuery(query) - .then(docs => { - return this.localStore - .remoteDocumentKeys(queryData.targetId) - .then(remoteKeys => { - const view = new View(query, remoteKeys); - const viewDocChanges = view.computeDocChanges(docs); - const viewChange = view.applyChanges(viewDocChanges); - assert( - viewChange.limboChanges.length === 0, - 'View returned limbo docs before target ack from the server.' - ); - assert( - !!viewChange.snapshot, - 'applyChanges for new view should always return a snapshot' - ); + .remoteDocumentKeys(queryData.targetId) + .then(remoteKeys => { + const view = new View(query, remoteKeys); + const viewDocChanges = view.computeDocChanges(docs); + const synthesizedTargetChange = TargetChange.createSynthesizedTargetChangeForCurrentChange( + queryData.targetId, + current && this.onlineState !== OnlineState.Offline + ); + const viewChange = view.applyChanges( + viewDocChanges, + /* updateLimboDocuments= */ this.isPrimary, + synthesizedTargetChange + ); + assert( + viewChange.limboChanges.length === 0, + 'View returned limbo docs before target ack from the server.' + ); + assert( + !!viewChange.snapshot, + 'applyChanges for new view should always return a snapshot' + ); - const data = new QueryView(query, queryData.targetId, view); - this.queryViewsByQuery.set(query, data); - this.queryViewsByTarget[queryData.targetId] = data; - this.viewHandler!([viewChange.snapshot!]); - this.remoteStore.listen(queryData); - }); - }) - .then(() => { - return queryData.targetId; + const data = new QueryView(query, queryData.targetId, view); + this.queryViewsByQuery.set(query, data); + this.queryViewsByTarget[queryData.targetId] = data; + return viewChange.snapshot!; + }); + }); + } + + /** + * Reconcile the list of synced documents in an existing view with those + * from persistence. + */ + // PORTING NOTE: Multi-tab only. + private synchronizeViewAndComputeSnapshot( + queryView: QueryView + ): Promise { + return this.localStore.executeQuery(queryView.query).then(docs => { + return this.localStore + .remoteDocumentKeys(queryView.targetId) + .then(async remoteKeys => { + const viewSnapshot = queryView.view.synchronizeWithPersistedState( + docs, + remoteKeys + ); + if (this.isPrimary) { + await this.updateTrackedLimbos( + queryView.targetId, + viewSnapshot.limboChanges + ); + } + return viewSnapshot; }); }); } /** Stops listening to the query. */ - unlisten(query: Query): Promise { + async unlisten(query: Query): Promise { this.assertSubscribed('unlisten()'); const queryView = this.queryViewsByQuery.get(query)!; assert(!!queryView, 'Trying to unlisten on query not found:' + query); - return this.localStore.releaseQuery(query).then(() => { - this.remoteStore.unlisten(queryView.targetId); - return this.removeAndCleanupQuery(queryView).then(() => { - return this.localStore.collectGarbage(); - }); - }); + if (this.isPrimary) { + // We need to remove the local query target first to allow us to verify + // whether any other client is still interested in this target. + this.sharedClientState.removeLocalQueryTarget(queryView.targetId); + const targetRemainsActive = this.sharedClientState.isActiveQueryTarget( + queryView.targetId + ); + + if (!targetRemainsActive) { + this.remoteStore.unlisten(queryView.targetId); + await this.localStore + .releaseQuery(query, /*keepPersistedQueryData=*/ false) + .then(() => this.removeAndCleanupQuery(queryView)) + .then(() => this.localStore.collectGarbage()) + .catch(err => this.ignoreIfPrimaryLeaseLoss(err)); + } + } else { + await this.removeAndCleanupQuery(queryView); + await this.localStore.releaseQuery( + query, + /*keepPersistedQueryData=*/ true + ); + } } /** @@ -232,6 +349,7 @@ export class SyncEngine implements RemoteSyncer { return this.localStore .localWrite(batch) .then(result => { + this.sharedClientState.addLocalPendingMutation(result.batchId); this.addMutationCallback(result.batchId, userCallback); return this.emitNewSnapsAndNotifyLocalStore(result.changes); }) @@ -306,64 +424,91 @@ export class SyncEngine implements RemoteSyncer { applyRemoteEvent(remoteEvent: RemoteEvent): Promise { this.assertSubscribed('applyRemoteEvent()'); - // Update `receivedDocument` as appropriate for any limbo targets. - objUtils.forEach(remoteEvent.targetChanges, (targetId, targetChange) => { - const limboResolution = this.limboResolutionsByTarget[targetId]; - if (limboResolution) { - // Since this is a limbo resolution lookup, it's for a single document - // and it could be added, modified, or removed, but not a combination. - assert( - targetChange.addedDocuments.size + - targetChange.modifiedDocuments.size + - targetChange.removedDocuments.size <= - 1, - 'Limbo resolution for single document contains multiple changes.' + return this.localStore + .applyRemoteEvent(remoteEvent) + .then(changes => { + // Update `receivedDocument` as appropriate for any limbo targets. + objUtils.forEach( + remoteEvent.targetChanges, + (targetId, targetChange) => { + const limboResolution = this.limboResolutionsByTarget[targetId]; + if (limboResolution) { + // Since this is a limbo resolution lookup, it's for a single document + // and it could be added, modified, or removed, but not a combination. + assert( + targetChange.addedDocuments.size + + targetChange.modifiedDocuments.size + + targetChange.removedDocuments.size <= + 1, + 'Limbo resolution for single document contains multiple changes.' + ); + if (targetChange.addedDocuments.size > 0) { + limboResolution.receivedDocument = true; + } else if (targetChange.modifiedDocuments.size > 0) { + assert( + limboResolution.receivedDocument, + 'Received change for limbo target document without add.' + ); + } else if (targetChange.removedDocuments.size > 0) { + assert( + limboResolution.receivedDocument, + 'Received remove for limbo target document without add.' + ); + limboResolution.receivedDocument = false; + } else { + // This was probably just a CURRENT targetChange or similar. + } + } + } ); - if (targetChange.addedDocuments.size > 0) { - limboResolution.receivedDocument = true; - } else if (targetChange.modifiedDocuments.size > 0) { - assert( - limboResolution.receivedDocument, - 'Received change for limbo target document without add.' - ); - } else if (targetChange.removedDocuments.size > 0) { - assert( - limboResolution.receivedDocument, - 'Received remove for limbo target document without add.' - ); - limboResolution.receivedDocument = false; - } else { - // This was probably just a CURRENT targetChange or similar. - } - } - }); - - return this.localStore.applyRemoteEvent(remoteEvent).then(changes => { - return this.emitNewSnapsAndNotifyLocalStore(changes, remoteEvent); - }); + return this.emitNewSnapsAndNotifyLocalStore(changes, remoteEvent); + }) + .catch(err => this.ignoreIfPrimaryLeaseLoss(err)); } /** * Applies an OnlineState change to the sync engine and notifies any views of * the change. */ - applyOnlineStateChange(onlineState: OnlineState): void { - const newViewSnapshots = [] as ViewSnapshot[]; - this.queryViewsByQuery.forEach((query, queryView) => { - const viewChange = queryView.view.applyOnlineStateChange(onlineState); - assert( - viewChange.limboChanges.length === 0, - 'OnlineState should not affect limbo documents.' - ); - if (viewChange.snapshot) { - newViewSnapshots.push(viewChange.snapshot); + applyOnlineStateChange( + onlineState: OnlineState, + source: OnlineStateSource + ): void { + // If we are the secondary client, we explicitly ignore the remote store's + // online state (the local client may go offline, even though the primary + // tab remains online) and only apply the primary tab's online state from + // SharedClientState. + if ( + (this.isPrimary && source === OnlineStateSource.RemoteStore) || + (!this.isPrimary && source === OnlineStateSource.SharedClientState) + ) { + const newViewSnapshots = [] as ViewSnapshot[]; + this.queryViewsByQuery.forEach((query, queryView) => { + const viewChange = queryView.view.applyOnlineStateChange(onlineState); + assert( + viewChange.limboChanges.length === 0, + 'OnlineState should not affect limbo documents.' + ); + if (viewChange.snapshot) { + newViewSnapshots.push(viewChange.snapshot); + } + }); + this.syncEngineListener!.onOnlineStateChange(onlineState); + this.syncEngineListener!.onWatchChange(newViewSnapshots); + + this.onlineState = onlineState; + if (this.isPrimary) { + this.sharedClientState.setOnlineState(onlineState); } - }); - this.viewHandler(newViewSnapshots); + } } - rejectListen(targetId: TargetId, err: FirestoreError): Promise { + async rejectListen(targetId: TargetId, err: FirestoreError): Promise { this.assertSubscribed('rejectListens()'); + + // PORTING NOTE: Multi-tab only. + this.sharedClientState.trackQueryUpdate(targetId, 'rejected', err); + const limboResolution = this.limboResolutionsByTarget[targetId]; const limboKey = limboResolution && limboResolution.key; if (limboKey) { @@ -398,33 +543,74 @@ export class SyncEngine implements RemoteSyncer { } else { const queryView = this.queryViewsByTarget[targetId]; assert(!!queryView, 'Unknown targetId: ' + targetId); - return this.localStore.releaseQuery(queryView.query).then(() => { - return this.removeAndCleanupQuery(queryView).then(() => { - this.errorHandler!(queryView.query, err); - }); - }); + await this.localStore + .releaseQuery(queryView.query, /* keepPersistedQueryData */ false) + .then(() => this.removeAndCleanupQuery(queryView)) + .catch(err => this.ignoreIfPrimaryLeaseLoss(err)); + this.syncEngineListener!.onWatchError(queryView.query, err); } } + // PORTING NOTE: Multi-tab only + async applyBatchState( + batchId: BatchId, + batchState: MutationBatchState, + error?: FirestoreError + ): Promise { + this.assertSubscribed('applyBatchState()'); + const documents = await this.localStore.lookupMutationDocuments(batchId); + + if (documents === null) { + // A throttled tab may not have seen the mutation before it was completed + // and removed from the mutation queue, in which case we won't have cached + // the affected documents. In this case we can safely ignore the update + // since that means we didn't apply the mutation locally at all (if we + // had, we would have cached the affected documents), and so we will just + // see any resulting document changes via normal remote document updates + // as applicable. + log.debug(LOG_TAG, 'Cannot apply mutation batch with id: ' + batchId); + return; + } + + if (batchState === 'pending') { + // If we are the primary client, we need to send this write to the + // backend. Secondary clients will ignore these writes since their remote + // connection is disabled. + await this.remoteStore.fillWritePipeline(); + } else if (batchState === 'acknowledged' || batchState === 'rejected') { + // NOTE: Both these methods are no-ops for batches that originated from + // other clients. + this.sharedClientState.removeLocalPendingMutation(batchId); + this.processUserCallback(batchId, error ? error : null); + + this.localStore.removeCachedMutationBatchMetadata(batchId); + } else { + fail(`Unknown batchState: ${batchState}`); + } + + await this.emitNewSnapsAndNotifyLocalStore(documents); + } + applySuccessfulWrite( mutationBatchResult: MutationBatchResult ): Promise { this.assertSubscribed('applySuccessfulWrite()'); + const batchId = mutationBatchResult.batch.batchId; + // The local store may or may not be able to apply the write result and // raise events immediately (depending on whether the watcher is caught // up), so we raise user callbacks first so that they consistently happen // before listen events. - this.processUserCallback( - mutationBatchResult.batch.batchId, - /*error=*/ null - ); + this.processUserCallback(batchId, /*error=*/ null); return this.localStore .acknowledgeBatch(mutationBatchResult) .then(changes => { + this.sharedClientState.removeLocalPendingMutation(batchId); return this.emitNewSnapsAndNotifyLocalStore(changes); - }); + }) + .catch(err => this.ignoreIfPrimaryLeaseLoss(err)); } rejectFailedWrite(batchId: BatchId, error: FirestoreError): Promise { @@ -436,9 +622,14 @@ export class SyncEngine implements RemoteSyncer { // listen events. this.processUserCallback(batchId, error); - return this.localStore.rejectBatch(batchId).then(changes => { - return this.emitNewSnapsAndNotifyLocalStore(changes); - }); + return this.localStore + .rejectBatch(batchId) + .then(changes => { + this.sharedClientState.trackMutationResult(batchId, 'rejected', error); + this.sharedClientState.removeLocalPendingMutation(batchId); + return this.emitNewSnapsAndNotifyLocalStore(changes); + }) + .catch(err => this.ignoreIfPrimaryLeaseLoss(err)); } private addMutationCallback( @@ -482,12 +673,16 @@ export class SyncEngine implements RemoteSyncer { } } - private removeAndCleanupQuery(queryView: QueryView): Promise { + private async removeAndCleanupQuery(queryView: QueryView): Promise { + this.sharedClientState.removeLocalQueryTarget(queryView.targetId); + this.queryViewsByQuery.delete(queryView.query); delete this.queryViewsByTarget[queryView.targetId]; - this.limboDocumentRefs.removeReferencesForId(queryView.targetId); - return this.gcLimboDocuments(); + if (this.isPrimary) { + this.limboDocumentRefs.removeReferencesForId(queryView.targetId); + await this.gcLimboDocuments(); + } } private updateTrackedLimbos( @@ -512,7 +707,7 @@ export class SyncEngine implements RemoteSyncer { const key = limboChange.key; if (!this.limboTargetsByKey.get(key)) { log.debug(LOG_TAG, 'New document in limbo: ' + key); - const limboTargetId = this.targetIdGenerator.next(); + const limboTargetId = this.limboTargetIdGenerator.next(); const query = Query.atPath(key.path); this.limboResolutionsByTarget[limboTargetId] = new LimboResolution(key); this.remoteStore.listen( @@ -550,7 +745,7 @@ export class SyncEngine implements RemoteSyncer { return this.limboTargetsByKey; } - private emitNewSnapsAndNotifyLocalStore( + private async emitNewSnapsAndNotifyLocalStore( changes: MaybeDocumentMap, remoteEvent?: RemoteEvent ): Promise { @@ -578,6 +773,7 @@ export class SyncEngine implements RemoteSyncer { remoteEvent && remoteEvent.targetChanges[queryView.targetId]; const viewChange = queryView.view.applyChanges( viewDocChanges, + /* updateLimboDocuments= */ this.isPrimary, targetChange ); return this.updateTrackedLimbos( @@ -585,6 +781,13 @@ export class SyncEngine implements RemoteSyncer { viewChange.limboChanges ).then(() => { if (viewChange.snapshot) { + if (this.isPrimary) { + this.sharedClientState.trackQueryUpdate( + queryView.targetId, + viewChange.snapshot.fromCache ? 'not-current' : 'current' + ); + } + newSnaps.push(viewChange.snapshot); const docChanges = LocalViewChanges.fromSnapshot( queryView.targetId, @@ -597,16 +800,38 @@ export class SyncEngine implements RemoteSyncer { ); }); - return Promise.all(queriesProcessed).then(() => { - this.viewHandler!(newSnaps); - this.localStore.notifyLocalViewChanges(docChangesInAllViews); - return this.localStore.collectGarbage(); - }); + await Promise.all(queriesProcessed); + this.syncEngineListener.onWatchChange(newSnaps); + this.localStore.notifyLocalViewChanges(docChangesInAllViews); + // TODO(multitab): Multitab garbage collection + if (this.isPrimary) { + await this.localStore + .collectGarbage() + .catch(err => this.ignoreIfPrimaryLeaseLoss(err)); + } + } + + /** + * Verifies the error thrown by an LocalStore operation. If a LocalStore + * operation fails because the primary lease has been taken by another client, + * we ignore the error (the persistence layer will immediately call + * `applyPrimaryLease` to propagate the primary state change). All other + * errors are re-thrown. + * + * @param err An error returned by a LocalStore operation. + * @return A Promise that resolves after we recovered, or the original error. + */ + private async ignoreIfPrimaryLeaseLoss(err: FirestoreError): Promise { + if (isPrimaryLeaseLostError(err)) { + log.debug(LOG_TAG, 'Unexpectedly lost primary lease'); + } else { + throw err; + } } private assertSubscribed(fnName: string): void { assert( - this.viewHandler !== null && this.errorHandler !== null, + this.syncEngineListener !== null, 'Trying to call ' + fnName + ' before calling subscribe().' ); } @@ -615,14 +840,214 @@ export class SyncEngine implements RemoteSyncer { this.currentUser = user; return this.localStore .handleUserChange(user) - .then(changes => { - return this.emitNewSnapsAndNotifyLocalStore(changes); + .then(result => { + this.sharedClientState.handleUserChange( + user, + result.removedBatchIds, + result.addedBatchIds + ); + return this.emitNewSnapsAndNotifyLocalStore(result.affectedDocuments); }) .then(() => { return this.remoteStore.handleUserChange(user); }); } + // PORTING NOTE: Multi-tab only + async applyPrimaryState(isPrimary: boolean): Promise { + if (isPrimary === true && this.isPrimary !== true) { + this.isPrimary = true; + await this.remoteStore.applyPrimaryState(true); + + // Secondary tabs only maintain Views for their local listeners and the + // Views internal state may not be 100% populated (in particular + // secondary tabs don't track syncedDocuments, the set of documents the + // server considers to be in the target). So when a secondary becomes + // primary, we need to need to make sure that all views for all targets + // match the state on disk. + const activeTargets = this.sharedClientState.getAllActiveQueryTargets(); + const activeQueries = await this.synchronizeQueryViewsAndRaiseSnapshots( + activeTargets.toArray() + ); + for (const queryData of activeQueries) { + this.remoteStore.listen(queryData); + } + } else if (isPrimary === false && this.isPrimary !== false) { + this.isPrimary = false; + const activeQueries = await this.synchronizeQueryViewsAndRaiseSnapshots( + objUtils.indices(this.queryViewsByTarget) + ); + this.resetLimboDocuments(); + for (const queryData of activeQueries) { + // TODO(multitab): Remove query views for non-local queries. + this.remoteStore.unlisten(queryData.targetId); + } + await this.remoteStore.applyPrimaryState(false); + } + } + + // PORTING NOTE: Multi-tab only. + private resetLimboDocuments(): void { + objUtils.forEachNumber(this.limboResolutionsByTarget, targetId => { + this.remoteStore.unlisten(targetId); + }); + this.limboResolutionsByTarget = []; + this.limboTargetsByKey = new SortedMap( + DocumentKey.comparator + ); + } + + /** + * Reconcile the query views of the provided query targets with the state from + * persistence. Raises snapshots for any changes that affect the local + * client and returns the updated state of all target's query data. + */ + // PORTING NOTE: Multi-tab only. + private synchronizeQueryViewsAndRaiseSnapshots( + targets: TargetId[] + ): Promise { + let p = Promise.resolve(); + const activeQueries: QueryData[] = []; + const newViewSnapshots: ViewSnapshot[] = []; + for (const targetId of targets) { + p = p.then(async () => { + let queryData: QueryData; + const queryView = this.queryViewsByTarget[targetId]; + if (queryView) { + // For queries that have a local View, we need to update their state + // in LocalStore (as the resume token and the snapshot version + // might have changed) and reconcile their views with the persisted + // state (the list of syncedDocuments may have gotten out of sync). + await this.localStore.releaseQuery( + queryView.query, + /*keepPersistedQueryData=*/ true + ); + queryData = await this.localStore.allocateQuery(queryView.query); + const viewChange = await this.synchronizeViewAndComputeSnapshot( + queryView + ); + if (viewChange.snapshot) { + newViewSnapshots.push(viewChange.snapshot); + } + } else { + assert( + this.isPrimary, + 'A secondary tab should never have an active query without an active view.' + ); + // For queries that never executed on this client, we need to + // allocate the query in LocalStore and initialize a new View. + const query = await this.localStore.getQueryForTarget(targetId); + queryData = await this.localStore.allocateQuery(query); + await this.initializeViewAndComputeSnapshot( + queryData, + /*current=*/ false + ); + } + activeQueries.push(queryData); + }); + } + return p.then(() => { + this.syncEngineListener!.onWatchChange(newViewSnapshots); + return activeQueries; + }); + } + + // PORTING NOTE: Multi-tab only + getActiveClients(): Promise { + return this.localStore.getActiveClients(); + } + + // PORTING NOTE: Multi-tab only + async applyTargetState( + targetId: TargetId, + state: QueryTargetState, + error?: FirestoreError + ): Promise { + if (this.isPrimary) { + // If we receive a target state notification via WebStorage, we are + // either already secondary or another tab has taken the primary lease. + log.debug(LOG_TAG, 'Ignoring unexpected query state notification.'); + return; + } + + if (this.queryViewsByTarget[targetId]) { + switch (state) { + case 'current': + case 'not-current': { + const changes = await this.localStore.getNewDocumentChanges(); + const synthesizedRemoteEvent = RemoteEvent.createSynthesizedRemoteEventForCurrentChange( + targetId, + state === 'current' + ); + return this.emitNewSnapsAndNotifyLocalStore( + changes, + synthesizedRemoteEvent + ); + } + case 'rejected': { + const queryView = this.queryViewsByTarget[targetId]; + await this.removeAndCleanupQuery(queryView); + await this.localStore.releaseQuery( + queryView.query, + /*keepPersistedQueryData=*/ true + ); + this.syncEngineListener!.onWatchError(queryView.query, error); + break; + } + default: + fail('Unexpected target state: ' + state); + } + } + } + + // PORTING NOTE: Multi-tab only + async applyActiveTargetsChange( + added: TargetId[], + removed: TargetId[] + ): Promise { + if (!this.isPrimary) { + return; + } + + for (const targetId of added) { + assert( + !this.queryViewsByTarget[targetId], + 'Trying to add an already active target' + ); + const query = await this.localStore.getQueryForTarget(targetId); + assert(!!query, `Query data for active target ${targetId} not found`); + const queryData = await this.localStore.allocateQuery(query); + await this.initializeViewAndComputeSnapshot( + queryData, + /*current=*/ false + ); + this.remoteStore.listen(queryData); + } + + for (const targetId of removed) { + const queryView = this.queryViewsByTarget[targetId]; + // Check that the query is still active since the query might have been + // removed if it has been rejected by the backend. + if (queryView) { + this.remoteStore.unlisten(targetId); + await this.localStore + .releaseQuery(queryView.query, /*keepPersistedQueryData=*/ false) + .then(() => this.removeAndCleanupQuery(queryView)) + .catch(err => this.ignoreIfPrimaryLeaseLoss(err)); + } + } + } + + enableNetwork(): Promise { + this.localStore.setNetworkEnabled(true); + return this.remoteStore.enableNetwork(); + } + + disableNetwork(): Promise { + this.localStore.setNetworkEnabled(false); + return this.remoteStore.disableNetwork(); + } + getRemoteKeysForTarget(targetId: TargetId): DocumentKeySet { const limboResolution = this.limboResolutionsByTarget[targetId]; if (limboResolution && limboResolution.receivedDocument) { diff --git a/packages/firestore/src/core/target_id_generator.ts b/packages/firestore/src/core/target_id_generator.ts index 937d1a59c9d..9a76ee08516 100644 --- a/packages/firestore/src/core/target_id_generator.ts +++ b/packages/firestore/src/core/target_id_generator.ts @@ -15,60 +15,80 @@ */ import { TargetId } from './types'; +import { assert } from '../util/assert'; const RESERVED_BITS = 1; enum GeneratorIds { - LocalStore = 0, - SyncEngine = 1 + QueryCache = 0, // The target IDs for user-issued queries are even (end in 0). + SyncEngine = 1 // The target IDs for limbo detection are odd (end in 1). } /** - * TargetIdGenerator generates monotonically increasing integer IDs. There are - * separate generators for different scopes. While these generators will operate - * independently of each other, they are scoped, such that no two generators - * will ever produce the same ID. This is useful, because sometimes the backend - * may group IDs from separate parts of the client into the same ID space. + * Generates monotonically increasing target IDs for sending targets to the + * watch stream. + * + * The client constructs two generators, one for the query cache (via + * forQueryCache()), and one for limbo documents (via forSyncEngine()). These + * two generators produce non-overlapping IDs (by using even and odd IDs + * respectively). + * + * By separating the target ID space, the query cache can generate target IDs + * that persist across client restarts, while sync engine can independently + * generate in-memory target IDs that are transient and can be reused after a + * restart. */ +// TODO(mrschmidt): Explore removing this class in favor of generating these IDs +// directly in SyncEngine and LocalStore. export class TargetIdGenerator { - private previousId: TargetId; + private nextId: TargetId; - constructor(private generatorId: number, initAfter: TargetId = 0) { - // Replace the generator part of initAfter with this generator's ID. - const afterWithoutGenerator = (initAfter >> RESERVED_BITS) << RESERVED_BITS; - const afterGenerator = initAfter - afterWithoutGenerator; - if (afterGenerator >= generatorId) { - // For example, if: - // this.generatorId = 0b0000 - // after = 0b1011 - // afterGenerator = 0b0001 - // Then: - // previous = 0b1010 - // next = 0b1100 - this.previousId = afterWithoutGenerator | this.generatorId; - } else { - // For example, if: - // this.generatorId = 0b0001 - // after = 0b1010 - // afterGenerator = 0b0000 - // Then: - // previous = 0b1001 - // next = 0b1011 - this.previousId = - (afterWithoutGenerator | this.generatorId) - (1 << RESERVED_BITS); - } + /** + * Instantiates a new TargetIdGenerator. If a seed is provided, the generator + * will use the seed value as the next target ID. + */ + constructor(private generatorId: number, seed?: number) { + assert( + (generatorId & RESERVED_BITS) === generatorId, + `Generator ID ${generatorId} contains more than ${RESERVED_BITS} reserved bits` + ); + this.seek(seed !== undefined ? seed : this.generatorId); } next(): TargetId { - this.previousId += 1 << RESERVED_BITS; - return this.previousId; + const nextId = this.nextId; + this.nextId += 1 << RESERVED_BITS; + return nextId; + } + + /** + * Returns the ID that follows the given ID. Subsequent calls to `next()` + * use the newly returned target ID as their base. + */ + after(targetId: TargetId): TargetId { + this.seek(targetId + (1 << RESERVED_BITS)); + return this.next(); + } + + private seek(targetId: TargetId): void { + assert( + (targetId & RESERVED_BITS) === this.generatorId, + 'Cannot supply target ID from different generator ID' + ); + this.nextId = targetId; } - static forLocalStore(initAfter: TargetId = 0): TargetIdGenerator { - return new TargetIdGenerator(GeneratorIds.LocalStore, initAfter); + static forQueryCache(): TargetIdGenerator { + // We seed the query cache generator to return '2' as its first ID, as there + // is no differentiation in the protocol layer between an unset number and + // the number '0'. If we were to sent a target with target ID '0', the + // backend would consider it unset and replace it with its own ID. + const targetIdGenerator = new TargetIdGenerator(GeneratorIds.QueryCache, 2); + return targetIdGenerator; } static forSyncEngine(): TargetIdGenerator { + // Sync engine assigns target IDs for limbo document detection. return new TargetIdGenerator(GeneratorIds.SyncEngine); } } diff --git a/packages/firestore/src/core/types.ts b/packages/firestore/src/core/types.ts index c5ef83a76bc..7e0b7c01f30 100644 --- a/packages/firestore/src/core/types.ts +++ b/packages/firestore/src/core/types.ts @@ -30,6 +30,9 @@ export type TargetId = number; // they're strings. We should probably (de-)serialize to a common internal type. export type ProtoByteString = Uint8Array | string; +/** The different states of a mutation batch. */ +export type MutationBatchState = 'pending' | 'acknowledged' | 'rejected'; + /** * Describes the online state of the Firestore client. Note that this does not * indicate whether or not the remote store is trying to connect or not. This is @@ -60,3 +63,9 @@ export enum OnlineState { */ Offline } + +/** The source of an online state event. */ +export enum OnlineStateSource { + RemoteStore, + SharedClientState +} diff --git a/packages/firestore/src/core/view.ts b/packages/firestore/src/core/view.ts index 91cb963e523..307ca2877f6 100644 --- a/packages/firestore/src/core/view.ts +++ b/packages/firestore/src/core/view.ts @@ -223,15 +223,19 @@ export class View { } /** - * Updates the view with the given ViewDocumentChanges and updates limbo docs - * and sync state from the given (optional) target change. + * Updates the view with the given ViewDocumentChanges and optionally updates + * limbo docs and sync state from the provided target change. * @param docChanges The set of changes to make to the view's docs. + * @param updateLimboDocuments Whether to update limbo documents based on this + * change. * @param targetChange A target change to apply for computing limbo docs and * sync state. * @return A new ViewChange with the given docs, changes, and sync state. */ + // PORTING NOTE: The iOS/Android clients always compute limbo document changes. applyChanges( docChanges: ViewDocumentChanges, + updateLimboDocuments: boolean, targetChange?: TargetChange ): ViewChange { assert(!docChanges.needsRefill, 'Cannot apply changes that need a refill'); @@ -248,7 +252,9 @@ export class View { }); this.applyTargetChange(targetChange); - const limboChanges = this.updateLimboDocuments(); + const limboChanges = updateLimboDocuments + ? this.updateLimboDocuments() + : []; const synced = this.limboDocuments.size === 0 && this.current; const newSyncState = synced ? SyncState.Synced : SyncState.Local; const syncStateChanged = newSyncState !== this.syncState; @@ -286,12 +292,15 @@ export class View { // are guaranteed to get a new TargetChange that sets `current` back to // true once the client is back online. this.current = false; - return this.applyChanges({ - documentSet: this.documentSet, - changeSet: new DocumentChangeSet(), - mutatedKeys: this.mutatedKeys, - needsRefill: false - }); + return this.applyChanges( + { + documentSet: this.documentSet, + changeSet: new DocumentChangeSet(), + mutatedKeys: this.mutatedKeys, + needsRefill: false + }, + /* updateLimboDocuments= */ false + ); } else { // No effect, just return a no-op ViewChange. return { limboChanges: [] }; @@ -373,6 +382,51 @@ export class View { }); return changes; } + + /** + * Update the in-memory state of the current view with the state read from + * persistence. + * + * We update the query view whenever a client's primary status changes: + * - When a client transitions from primary to secondary, it can miss + * LocalStorage updates and its query views may temporarily not be + * synchronized with the state on disk. + * - For secondary to primary transitions, the client needs to update the list + * of `syncedDocuments` since secondary clients update their query views + * based purely on synthesized RemoteEvents. + * + * @param localDocs - The documents that match the query according to the + * LocalStore. + * @param remoteKeys - The keys of the documents that match the query + * according to the backend. + * + * @return The ViewChange that resulted from this synchronization. + */ + // PORTING NOTE: Multi-tab only. + synchronizeWithPersistedState( + localDocs: MaybeDocumentMap, + remoteKeys: DocumentKeySet + ): ViewChange { + this._syncedDocuments = remoteKeys; + this.limboDocuments = documentKeySet(); + const docChanges = this.computeDocChanges(localDocs); + return this.applyChanges(docChanges, /*updateLimboDocuments=*/ true); + } + + /** + * Returns a view snapshot as if this query was just listened to. Contains + * a document add for every existing document and the `fromCache` and + * `hasPendingWrites` status of the already established view. + */ + // PORTING NOTE: Multi-tab only. + computeInitialSnapshot(): ViewSnapshot { + return ViewSnapshot.fromInitialDocuments( + this.query, + this.documentSet, + this.syncState === SyncState.Local, + !this.mutatedKeys.isEmpty() + ); + } } function compareChangeType(c1: ChangeType, c2: ChangeType): number { diff --git a/packages/firestore/src/core/view_snapshot.ts b/packages/firestore/src/core/view_snapshot.ts index c5b625a6fd3..2a34e4247d6 100644 --- a/packages/firestore/src/core/view_snapshot.ts +++ b/packages/firestore/src/core/view_snapshot.ts @@ -149,6 +149,30 @@ export class ViewSnapshot { readonly excludesMetadataChanges: boolean ) {} + /** Returns a view snapshot as if all documents in the snapshot were added. */ + static fromInitialDocuments( + query: Query, + documents: DocumentSet, + fromCache: boolean, + hasPendingWrites: boolean + ): ViewSnapshot { + const changes: DocumentViewChange[] = []; + documents.forEach(doc => { + changes.push({ type: ChangeType.Added, doc }); + }); + + return new ViewSnapshot( + query, + documents, + DocumentSet.emptySet(documents), + changes, + fromCache, + hasPendingWrites, + /* syncStateChanged */ true, + /* excludesMetadataChanges= */ false + ); + } + isEqual(other: ViewSnapshot): boolean { if ( this.fromCache !== other.fromCache || diff --git a/packages/firestore/src/local/indexeddb_mutation_queue.ts b/packages/firestore/src/local/indexeddb_mutation_queue.ts index a97fd8b0f78..cd19c0078bc 100644 --- a/packages/firestore/src/local/indexeddb_mutation_queue.ts +++ b/packages/firestore/src/local/indexeddb_mutation_queue.ts @@ -24,7 +24,7 @@ import { Mutation } from '../model/mutation'; import { BATCHID_UNKNOWN, MutationBatch } from '../model/mutation_batch'; import { ResourcePath } from '../model/path'; import { assert, fail } from '../util/assert'; -import { immediatePredecessor, primitiveComparator } from '../util/misc'; +import { primitiveComparator } from '../util/misc'; import { SortedSet } from '../util/sorted_set'; import * as EncodedResourcePath from './encoded_resource_path'; @@ -47,18 +47,18 @@ import { IndexedDbPersistence } from './indexeddb_persistence'; /** A mutation queue for a specific user, backed by IndexedDB. */ export class IndexedDbMutationQueue implements MutationQueue { /** - * Next value to use when assigning sequential IDs to each mutation batch. + * Caches the document keys for pending mutation batches. If the mutation + * has been removed from IndexedDb, the cached value may continue to + * be used to retrieve the batch's document keys. To remove a cached value + * locally, `removeCachedMutationKeys()` should be invoked either directly + * or through `removeMutationBatches()`. * - * NOTE: There can only be one IndexedDbMutationQueue for a given db at a - * time, hence it is safe to track nextBatchID as an instance-level property. - * Should we ever relax this constraint we'll need to revisit this. + * With multi-tab, when the primary client acknowledges or rejects a mutation, + * this cache is used by secondary clients to invalidate the local + * view of the documents that were previously affected by the mutation. */ - private nextBatchId: BatchId; - - /** - * A write-through cache copy of the metadata describing the current queue. - */ - private metadata: DbMutationQueue; + // PORTING NOTE: Multi-tab only. + private documentKeysByBatchId = {} as { [batchId: number]: DocumentKeySet }; private garbageCollector: GarbageCollector | null = null; @@ -90,94 +90,34 @@ export class IndexedDbMutationQueue implements MutationQueue { } start(transaction: PersistenceTransaction): PersistencePromise { - return IndexedDbMutationQueue.loadNextBatchIdFromDb(transaction) - .next(nextBatchId => { - this.nextBatchId = nextBatchId; - return mutationQueuesStore(transaction).get(this.userId); - }) - .next((metadata: DbMutationQueue | null) => { - if (!metadata) { - metadata = new DbMutationQueue( - this.userId, - BATCHID_UNKNOWN, - /*lastStreamToken=*/ '' - ); - } - this.metadata = metadata; - - // On restart, nextBatchId may end up lower than - // lastAcknowledgedBatchId since it's computed from the queue - // contents, and there may be no mutations in the queue. In this - // case, we need to reset lastAcknowledgedBatchId (which is safe - // since the queue must be empty). - if (this.metadata.lastAcknowledgedBatchId >= this.nextBatchId) { - return this.checkEmpty(transaction).next(empty => { - assert( - empty, - 'Reset nextBatchID is only possible when the queue is empty' - ); - - this.metadata.lastAcknowledgedBatchId = BATCHID_UNKNOWN; - return mutationQueuesStore(transaction).put(this.metadata); - }); - } else { - return PersistencePromise.resolve(); - } - }); - } - - /** - * Returns one larger than the largest batch ID that has been stored. If there - * are no mutations returns 0. Note that batch IDs are global. - */ - static loadNextBatchIdFromDb( - txn: PersistenceTransaction - ): PersistencePromise { - let maxBatchId = BATCHID_UNKNOWN; - return mutationsStore(txn) - .iterate({ reverse: true }, (key, batch, control) => { - const [userId, batchId] = key; - if (batchId > maxBatchId) { - maxBatchId = batch.batchId; - } - - if (userId === '') { - // We can't compute a predecessor for the empty string, since it - // is lexographically first. That also means that no other - // userIds can come before this one, so we can just exit early. - control.done(); - } else { - const nextUser = immediatePredecessor(userId); - control.skip([nextUser]); - } - }) - .next(() => maxBatchId + 1); + // PORTING NOTE: On the Web, mutation batch IDs are auto-generated by + // IndexedDb, which guarantees their uniqueness across all users. + return PersistencePromise.resolve(); } checkEmpty(transaction: PersistenceTransaction): PersistencePromise { let empty = true; const range = IDBKeyRange.bound( - this.keyForBatchId(Number.NEGATIVE_INFINITY), - this.keyForBatchId(Number.POSITIVE_INFINITY) + [this.userId, Number.NEGATIVE_INFINITY], + [this.userId, Number.POSITIVE_INFINITY] ); return mutationsStore(transaction) - .iterate({ range }, (key, value, control) => { - empty = false; - control.done(); - }) + .iterate( + { index: DbMutationBatch.userMutationsIndex, range }, + (key, value, control) => { + empty = false; + control.done(); + } + ) .next(() => empty); } - getNextBatchId( - transaction: PersistenceTransaction - ): PersistencePromise { - return PersistencePromise.resolve(this.nextBatchId); - } - getHighestAcknowledgedBatchId( transaction: PersistenceTransaction ): PersistencePromise { - return PersistencePromise.resolve(this.metadata.lastAcknowledgedBatchId); + return this.getMutationQueueMetadata(transaction).next(metadata => { + return metadata.lastAcknowledgedBatchId; + }); } acknowledgeBatch( @@ -185,23 +125,25 @@ export class IndexedDbMutationQueue implements MutationQueue { batch: MutationBatch, streamToken: ProtoByteString ): PersistencePromise { - const batchId = batch.batchId; - assert( - batchId > this.metadata.lastAcknowledgedBatchId, - 'Mutation batchIDs must be acknowledged in order' - ); + return this.getMutationQueueMetadata(transaction).next(metadata => { + const batchId = batch.batchId; + assert( + batchId > metadata.lastAcknowledgedBatchId, + 'Mutation batchIDs must be acknowledged in order' + ); - this.metadata.lastAcknowledgedBatchId = batchId; - this.metadata.lastStreamToken = convertStreamToken(streamToken); + metadata.lastAcknowledgedBatchId = batchId; + metadata.lastStreamToken = convertStreamToken(streamToken); - return mutationQueuesStore(transaction).put(this.metadata); + return mutationQueuesStore(transaction).put(metadata); + }); } getLastStreamToken( transaction: PersistenceTransaction ): PersistencePromise { - return PersistencePromise.resolve( - this.metadata.lastStreamToken + return this.getMutationQueueMetadata(transaction).next( + metadata => metadata.lastStreamToken ); } @@ -209,8 +151,10 @@ export class IndexedDbMutationQueue implements MutationQueue { transaction: PersistenceTransaction, streamToken: ProtoByteString ): PersistencePromise { - this.metadata.lastStreamToken = convertStreamToken(streamToken); - return mutationQueuesStore(transaction).put(this.metadata); + return this.getMutationQueueMetadata(transaction).next(metadata => { + metadata.lastStreamToken = convertStreamToken(streamToken); + return mutationQueuesStore(transaction).put(metadata); + }); } addMutationBatch( @@ -218,86 +162,117 @@ export class IndexedDbMutationQueue implements MutationQueue { localWriteTime: Timestamp, mutations: Mutation[] ): PersistencePromise { - const batchId = this.nextBatchId; - this.nextBatchId++; - const batch = new MutationBatch(batchId, localWriteTime, mutations); + const documentStore = documentMutationsStore(transaction); + const mutationStore = mutationsStore(transaction); + + // The IndexedDb implementation in Chrome (and Firefox) does not handle + // compound indices that include auto-generated keys correctly. To ensure + // that the index entry is added correctly in all browsers, we perform two + // writes: The first write is used to retrieve the next auto-generated Batch + // ID, and the second write populates the index and stores the actual + // mutation batch. + // See: https://bugs.chromium.org/p/chromium/issues/detail?id=701972 + + // tslint:disable-next-line:no-any We write an empty object to obtain key + return mutationStore.add({} as any).next(batchId => { + assert(typeof batchId === 'number', 'Auto-generated key is not a number'); + + const batch = new MutationBatch(batchId, localWriteTime, mutations); + const dbBatch = this.serializer.toDbMutationBatch(this.userId, batch); - const dbBatch = this.serializer.toDbMutationBatch(this.userId, batch); + this.documentKeysByBatchId[batchId] = batch.keys(); + + const promises: Array> = []; + for (const mutation of mutations) { + const indexKey = DbDocumentMutation.key( + this.userId, + mutation.key.path, + batchId + ); + promises.push(mutationStore.put(dbBatch)); + promises.push( + documentStore.put(indexKey, DbDocumentMutation.PLACEHOLDER) + ); + } + return PersistencePromise.waitFor(promises).next(() => batch); + }); + } + lookupMutationBatch( + transaction: PersistenceTransaction, + batchId: BatchId + ): PersistencePromise { return mutationsStore(transaction) - .put(dbBatch) - .next(() => { - const promises: Array> = []; - for (const mutation of mutations) { - const indexKey = DbDocumentMutation.key( - this.userId, - mutation.key.path, - batchId - ); - promises.push( - documentMutationsStore(transaction).put( - indexKey, - DbDocumentMutation.PLACEHOLDER - ) + .get(batchId) + .next(dbBatch => { + if (dbBatch) { + assert( + dbBatch.userId === this.userId, + `Unexpected user '${dbBatch.userId}' for mutation batch ${batchId}` ); + return this.serializer.fromDbMutationBatch(dbBatch); } - return PersistencePromise.waitFor(promises); - }) - .next(() => { - return batch; + return null; }); } - lookupMutationBatch( + lookupMutationKeys( transaction: PersistenceTransaction, batchId: BatchId - ): PersistencePromise { - return mutationsStore(transaction) - .get(this.keyForBatchId(batchId)) - .next( - dbBatch => - dbBatch ? this.serializer.fromDbMutationBatch(dbBatch) : null - ); + ): PersistencePromise { + if (this.documentKeysByBatchId[batchId]) { + return PersistencePromise.resolve(this.documentKeysByBatchId[batchId]); + } else { + return this.lookupMutationBatch(transaction, batchId).next(batch => { + if (batch) { + const keys = batch.keys(); + this.documentKeysByBatchId[batchId] = keys; + return keys; + } else { + return null; + } + }); + } } getNextMutationBatchAfterBatchId( transaction: PersistenceTransaction, batchId: BatchId ): PersistencePromise { - // All batches with batchId <= this.metadata.lastAcknowledgedBatchId have - // been acknowledged so the first unacknowledged batch after batchID will - // have a batchID larger than both of these values. - const nextBatchId = - Math.max(batchId, this.metadata.lastAcknowledgedBatchId) + 1; - - const range = IDBKeyRange.lowerBound(this.keyForBatchId(nextBatchId)); - let foundBatch: MutationBatch | null = null; - return mutationsStore(transaction) - .iterate({ range }, (key, dbBatch, control) => { - if (dbBatch.userId === this.userId) { - assert( - dbBatch.batchId >= nextBatchId, - 'Should have found mutation after ' + nextBatchId - ); - foundBatch = this.serializer.fromDbMutationBatch(dbBatch); - } - control.done(); - }) - .next(() => foundBatch); + return this.getMutationQueueMetadata(transaction).next(metadata => { + // All batches with batchId <= this.metadata.lastAcknowledgedBatchId have + // been acknowledged so the first unacknowledged batch after batchID will + // have a batchID larger than both of these values. + const nextBatchId = + Math.max(batchId, metadata.lastAcknowledgedBatchId) + 1; + + const range = IDBKeyRange.lowerBound([this.userId, nextBatchId]); + let foundBatch: MutationBatch | null = null; + return mutationsStore(transaction) + .iterate( + { index: DbMutationBatch.userMutationsIndex, range }, + (key, dbBatch, control) => { + if (dbBatch.userId === this.userId) { + assert( + dbBatch.batchId >= nextBatchId, + 'Should have found mutation after ' + nextBatchId + ); + foundBatch = this.serializer.fromDbMutationBatch(dbBatch); + } + control.done(); + } + ) + .next(() => foundBatch); + }); } getAllMutationBatches( transaction: PersistenceTransaction ): PersistencePromise { - const range = IDBKeyRange.bound( - this.keyForBatchId(BATCHID_UNKNOWN), - this.keyForBatchId(Number.POSITIVE_INFINITY) + return this.getAllMutationBatchesThroughBatchId( + transaction, + Number.POSITIVE_INFINITY ); - return mutationsStore(transaction) - .loadAll(range) - .next(dbBatches => - dbBatches.map(dbBatch => this.serializer.fromDbMutationBatch(dbBatch)) - ); } getAllMutationBatchesThroughBatchId( @@ -305,11 +280,11 @@ export class IndexedDbMutationQueue implements MutationQueue { batchId: BatchId ): PersistencePromise { const range = IDBKeyRange.bound( - this.keyForBatchId(BATCHID_UNKNOWN), - this.keyForBatchId(batchId) + [this.userId, BATCHID_UNKNOWN], + [this.userId, batchId] ); return mutationsStore(transaction) - .loadAll(range) + .loadAll(DbMutationBatch.userMutationsIndex, range) .next(dbBatches => dbBatches.map(dbBatch => this.serializer.fromDbMutationBatch(dbBatch)) ); @@ -330,7 +305,7 @@ export class IndexedDbMutationQueue implements MutationQueue { const results: MutationBatch[] = []; return documentMutationsStore(transaction) .iterate({ range: indexStart }, (indexKey, _, control) => { - const [userID, encodedPath, batchID] = indexKey; + const [userID, encodedPath, batchId] = indexKey; // Only consider rows matching exactly the specific key of // interest. Note that because we order by path first, and we @@ -344,23 +319,25 @@ export class IndexedDbMutationQueue implements MutationQueue { control.done(); return; } - const mutationKey = this.keyForBatchId(batchID); // Look up the mutation batch in the store. - // PORTING NOTE: because iteration is callback driven in the web, - // we just look up the key instead of keeping an open iterator - // like iOS. return mutationsStore(transaction) - .get(mutationKey) - .next(dbBatch => { - if (dbBatch === null) { + .get(batchId) + .next(mutation => { + if (!mutation) { fail( 'Dangling document-mutation reference found: ' + indexKey + ' which points to ' + - mutationKey + batchId ); } - results.push(this.serializer.fromDbMutationBatch(dbBatch!)); + assert( + mutation.userId === this.userId, + `Unexpected user '${ + mutation.userId + }' for mutation batch ${batchId}` + ); + results.push(this.serializer.fromDbMutationBatch(mutation!)); }); }) .next(() => results); @@ -471,19 +448,24 @@ export class IndexedDbMutationQueue implements MutationQueue { const results: MutationBatch[] = []; const promises: Array> = []; // TODO(rockwood): Implement this using iterate. - batchIDs.forEach(batchID => { - const mutationKey = this.keyForBatchId(batchID); + batchIDs.forEach(batchId => { promises.push( mutationsStore(transaction) - .get(mutationKey) + .get(batchId) .next(mutation => { if (mutation === null) { fail( 'Dangling document-mutation reference found, ' + 'which points to ' + - mutationKey + batchId ); } + assert( + mutation.userId === this.userId, + `Unexpected user '${ + mutation.userId + }' for mutation batch ${batchId}` + ); results.push(this.serializer.fromDbMutationBatch(mutation!)); }) ); @@ -495,17 +477,20 @@ export class IndexedDbMutationQueue implements MutationQueue { transaction: PersistenceTransaction, batches: MutationBatch[] ): PersistencePromise { - const txn = mutationsStore(transaction); + const mutationStore = mutationsStore(transaction); const indexTxn = documentMutationsStore(transaction); const promises: Array> = []; for (const batch of batches) { - const range = IDBKeyRange.only(this.keyForBatchId(batch.batchId)); + const range = IDBKeyRange.only(batch.batchId); let numDeleted = 0; - const removePromise = txn.iterate({ range }, (key, value, control) => { - numDeleted++; - return control.delete(); - }); + const removePromise = mutationStore.iterate( + { range }, + (key, value, control) => { + numDeleted++; + return control.delete(); + } + ); promises.push( removePromise.next(() => { assert( @@ -521,6 +506,7 @@ export class IndexedDbMutationQueue implements MutationQueue { mutation.key.path, batch.batchId ); + this.removeCachedMutationKeys(batch.batchId); promises.push(indexTxn.delete(indexKey)); if (this.garbageCollector !== null) { this.garbageCollector.addPotentialGarbageKey(mutation.key); @@ -530,6 +516,10 @@ export class IndexedDbMutationQueue implements MutationQueue { return PersistencePromise.waitFor(promises); } + removeCachedMutationKeys(batchId: BatchId): void { + delete this.documentKeysByBatchId[batchId]; + } + performConsistencyCheck( txn: PersistenceTransaction ): PersistencePromise { @@ -588,12 +578,23 @@ export class IndexedDbMutationQueue implements MutationQueue { .next(() => containsKey); } - /** - * Creates a [userId, batchId] key for use with the DbMutationQueue object - * store. - */ - private keyForBatchId(batchId: BatchId): DbMutationBatchKey { - return [this.userId, batchId]; + // PORTING NOTE: Multi-tab only (state is held in memory in other clients). + /** Returns the mutation queue's metadata from IndexedDb. */ + private getMutationQueueMetadata( + transaction: PersistenceTransaction + ): PersistencePromise { + return mutationQueuesStore(transaction) + .get(this.userId) + .next((metadata: DbMutationQueue | null) => { + return ( + metadata || + new DbMutationQueue( + this.userId, + BATCHID_UNKNOWN, + /*lastStreamToken=*/ '' + ) + ); + }); } } @@ -602,7 +603,7 @@ function convertStreamToken(token: ProtoByteString): string { // TODO(b/78771403): Convert tokens to strings during deserialization assert( process.env.USE_MOCK_PERSISTENCE === 'YES', - 'Persisting non-string stream tokens is only supported with mock persistence .' + 'Persisting non-string stream tokens is only supported with mock persistence.' ); return token.toString(); } else { diff --git a/packages/firestore/src/local/indexeddb_persistence.ts b/packages/firestore/src/local/indexeddb_persistence.ts index 3aa3ca68be0..e76205ac52c 100644 --- a/packages/firestore/src/local/indexeddb_persistence.ts +++ b/packages/firestore/src/local/indexeddb_persistence.ts @@ -20,7 +20,6 @@ import { JsonProtoSerializer } from '../remote/serializer'; import { assert, fail } from '../util/assert'; import { Code, FirestoreError } from '../util/error'; import * as log from '../util/log'; -import { AutoId } from '../util/misc'; import { IndexedDbMutationQueue } from './indexeddb_mutation_queue'; import { IndexedDbQueryCache } from './indexeddb_query_cache'; @@ -28,37 +27,62 @@ import { IndexedDbRemoteDocumentCache } from './indexeddb_remote_document_cache' import { ALL_STORES, createOrUpgradeDb, + DbClientMetadataKey, + DbClientMetadata, DbOwner, DbOwnerKey, SCHEMA_VERSION } from './indexeddb_schema'; import { LocalSerializer } from './local_serializer'; import { MutationQueue } from './mutation_queue'; -import { Persistence, PersistenceTransaction } from './persistence'; +import { + Persistence, + PersistenceTransaction, + PrimaryStateListener +} from './persistence'; import { PersistencePromise } from './persistence_promise'; import { QueryCache } from './query_cache'; import { RemoteDocumentCache } from './remote_document_cache'; import { SimpleDb, SimpleDbStore, SimpleDbTransaction } from './simple_db'; +import { Platform } from '../platform/platform'; +import { AsyncQueue, TimerId } from '../util/async_queue'; +import { ClientId } from './shared_client_state'; +import { CancelablePromise } from '../util/promise'; const LOG_TAG = 'IndexedDbPersistence'; -/** If the owner lease is older than 5 seconds, try to take ownership. */ -const OWNER_LEASE_MAX_AGE_MS = 5000; -/** Refresh the owner lease every 4 seconds while owner. */ -const OWNER_LEASE_REFRESH_INTERVAL_MS = 4000; - -/** LocalStorage location to indicate a zombied ownerId (see class comment). */ -const ZOMBIE_OWNER_LOCALSTORAGE_SUFFIX = 'zombiedOwnerId'; -/** Error when the owner lease cannot be acquired or is lost. */ -const EXISTING_OWNER_ERROR_MSG = - 'There is another tab open with offline' + - ' persistence enabled. Only one such tab is allowed at a time. The' + - ' other tab must be closed or persistence must be disabled.'; +/** + * Oldest acceptable age in milliseconds for client metadata read from + * IndexedDB. Client metadata and primary leases that are older than 5 seconds + * are ignored. + */ +const CLIENT_METADATA_MAX_AGE_MS = 5000; +/** + * The interval at which clients will update their metadata, including + * refreshing their primary lease if held or potentially trying to acquire it if + * not held. + * + * Primary clients may opportunistically refresh their metadata earlier + * if they're already performing an IndexedDB operation. + */ +const CLIENT_METADATA_REFRESH_INTERVAL_MS = 4000; +/** User-facing error when the primary lease is required but not available. */ +const PRIMARY_LEASE_LOST_ERROR_MSG = + 'The current tab is not in the required state to perform this operation. ' + + 'It might be necessary to refresh the browser tab.'; +const PRIMARY_LEASE_EXCLUSIVE_ERROR_MSG = + 'Another tab has exclusive access to the persistence layer. ' + + 'To allow shared access, make sure to invoke ' + + '`enablePersistence()` with `experimentalTabSynchronization:true` in all tabs.'; const UNSUPPORTED_PLATFORM_ERROR_MSG = 'This platform is either missing' + ' IndexedDB or is known to have an incomplete implementation. Offline' + ' persistence has been disabled.'; +// The format of the LocalStorage key that stores zombied client is: +// firestore_zombie__ +const ZOMBIED_CLIENTS_KEY_PREFIX = 'firestore_zombie'; + export class IndexedDbTransaction extends PersistenceTransaction { constructor(readonly simpleDbTransaction: SimpleDbTransaction) { super(); @@ -90,9 +114,11 @@ export class IndexedDbTransaction extends PersistenceTransaction { * a refreshed tab is able to immediately re-acquire the owner lease). * Unfortunately, IndexedDB cannot be reliably used in window.unload since it is * an asynchronous API. So in addition to attempting to give up the lease, - * the owner writes its ownerId to a "zombiedOwnerId" entry in LocalStorage + * the owner writes its ownerId to a "zombiedClientId" entry in LocalStorage * which acts as an indicator that another tab should go ahead and take the * owner lease immediately regardless of the current lease timestamp. + * + * TODO(multitab): Update this comment with multi-tab changes. */ export class IndexedDbPersistence implements Persistence { static getStore( @@ -112,32 +138,59 @@ export class IndexedDbPersistence implements Persistence { */ static MAIN_DATABASE = 'main'; + private readonly document: Document | null; + private readonly window: Window; + private simpleDb: SimpleDb; private _started = false; + private isPrimary = false; + private networkEnabled = true; private dbName: string; - private localStoragePrefix: string; - private ownerId: string = this.generateOwnerId(); /** * Set to an Error object if we encounter an unrecoverable error. All further * transactions will be failed with this error. */ private persistenceError: Error | null; - /** The setInterval() handle tied to refreshing the owner lease. */ - // tslint:disable-next-line:no-any setTimeout() type differs on browser / node - private ownerLeaseRefreshHandle: any; /** Our window.unload handler, if registered. */ private windowUnloadHandler: (() => void) | null; + private inForeground = false; private serializer: LocalSerializer; - constructor(prefix: string, serializer: JsonProtoSerializer) { - this.dbName = prefix + IndexedDbPersistence.MAIN_DATABASE; + /** Our 'visibilitychange' listener if registered. */ + private documentVisibilityHandler: ((e?: Event) => void) | null; + + /** The client metadata refresh task. */ + private clientMetadataRefresher: CancelablePromise; + + /** Whether to allow shared multi-tab access to the persistence layer. */ + private allowTabSynchronization: boolean; + + /** A listener to notify on primary state changes. */ + private primaryStateListener: PrimaryStateListener = _ => Promise.resolve(); + + constructor( + private readonly persistenceKey: string, + private readonly clientId: ClientId, + platform: Platform, + private readonly queue: AsyncQueue, + serializer: JsonProtoSerializer + ) { + this.dbName = persistenceKey + IndexedDbPersistence.MAIN_DATABASE; this.serializer = new LocalSerializer(serializer); - this.localStoragePrefix = prefix; + this.document = platform.document; + this.window = platform.window; } - start(): Promise { + /** + * Attempt to start IndexedDb persistence. + * + * @param {boolean} synchronizeTabs Whether to enable shared persistence + * across multiple tabs. + * @return {Promise} Whether persistence was enabled. + */ + start(synchronizeTabs?: boolean): Promise { if (!IndexedDbPersistence.isAvailable()) { this.persistenceError = new FirestoreError( Code.UNIMPLEMENTED, @@ -147,32 +200,256 @@ export class IndexedDbPersistence implements Persistence { } assert(!this.started, 'IndexedDbPersistence double-started!'); + this.allowTabSynchronization = !!synchronizeTabs; + + assert(this.window !== null, "Expected 'window' to be defined"); return SimpleDb.openOrCreate(this.dbName, SCHEMA_VERSION, createOrUpgradeDb) .then(db => { this.simpleDb = db; }) - .then(() => this.tryAcquireOwnerLease()) .then(() => { - this.scheduleOwnerLeaseRefreshes(); + this.attachVisibilityHandler(); this.attachWindowUnloadHook(); + return this.updateClientMetadataAndTryBecomePrimary().then(() => + this.scheduleClientMetadataAndPrimaryLeaseRefreshes() + ); }) .then(() => { this._started = true; }); } - shutdown(deleteData?: boolean): Promise { - assert(this.started, 'IndexedDbPersistence shutdown without start!'); + setPrimaryStateListener( + primaryStateListener: PrimaryStateListener + ): Promise { + this.primaryStateListener = primaryStateListener; + return primaryStateListener(this.isPrimary); + } + + setNetworkEnabled(networkEnabled: boolean): void { + if (this.networkEnabled !== networkEnabled) { + this.networkEnabled = networkEnabled; + // Schedule a primary lease refresh for immediate execution. The eventual + // lease update will be propagated via `primaryStateListener`. + this.queue.enqueueAndForget(async () => { + if (this.started) { + await this.updateClientMetadataAndTryBecomePrimary(); + } + }); + } + } + + /** + * Updates the client metadata in IndexedDb and attempts to either obtain or + * extend the primary lease for the local client. Asynchronously notifies the + * primary state listener if the client either newly obtained or released its + * primary lease. + */ + private updateClientMetadataAndTryBecomePrimary(): Promise { + return this.simpleDb.runTransaction('readwrite', ALL_STORES, txn => { + const metadataStore = clientMetadataStore(txn); + return metadataStore + .put( + new DbClientMetadata( + this.clientId, + Date.now(), + this.networkEnabled, + this.inForeground + ) + ) + .next(() => this.canActAsPrimary(txn)) + .next(canActAsPrimary => { + const wasPrimary = this.isPrimary; + this.isPrimary = canActAsPrimary; + + if (wasPrimary !== this.isPrimary) { + this.queue.enqueueAndForget(async () => { + // Verify that `shutdown()` hasn't been called yet by the time + // we invoke the `primaryStateListener`. + if (this.started) { + return this.primaryStateListener(this.isPrimary); + } + }); + } + + if (wasPrimary && !this.isPrimary) { + return this.releasePrimaryLeaseIfHeld(txn); + } else if (this.isPrimary) { + return this.acquireOrExtendPrimaryLease(txn); + } + }); + }); + } + + private removeClientMetadata( + txn: SimpleDbTransaction + ): PersistencePromise { + const metadataStore = clientMetadataStore(txn); + return metadataStore.delete(this.clientId); + } + + /** + * Schedules a recurring timer to update the client metadata and to either + * extend or acquire the primary lease if the client is eligible. + */ + private scheduleClientMetadataAndPrimaryLeaseRefreshes(): void { + this.clientMetadataRefresher = this.queue.enqueueAfterDelay( + TimerId.ClientMetadataRefresh, + CLIENT_METADATA_REFRESH_INTERVAL_MS, + () => { + return this.updateClientMetadataAndTryBecomePrimary().then(() => + this.scheduleClientMetadataAndPrimaryLeaseRefreshes() + ); + } + ); + } + + /** Checks whether `client` is the local client. */ + private isLocalClient(client: DbOwner | null): boolean { + return client ? client.ownerId === this.clientId : false; + } + + /** + * Evaluate the state of all active clients and determine whether the local + * client is or can act as the holder of the primary lease. Returns whether + * the client is eligible for the lease, but does not actually acquire it. + * May return 'false' even if there is no active leaseholder and another + * (foreground) client should become leaseholder instead. + */ + private canActAsPrimary( + txn: SimpleDbTransaction + ): PersistencePromise { + const store = ownerStore(txn); + return store + .get('owner') + .next(currentPrimary => { + const currentLeaseIsValid = + currentPrimary !== null && + this.isWithinMaxAge(currentPrimary.leaseTimestampMs) && + !this.isClientZombied(currentPrimary.ownerId); + + // A client is eligible for the primary lease if: + // - its network is enabled and the client's tab is in the foreground. + // - its network is enabled and no other client's tab is in the + // foreground. + // - every clients network is disabled and the client's tab is in the + // foreground. + // - every clients network is disabled and no other client's tab is in + // the foreground. + if (currentLeaseIsValid) { + if (this.isLocalClient(currentPrimary) && this.networkEnabled) { + return true; + } + + if (!this.isLocalClient(currentPrimary)) { + if (!currentPrimary.allowTabSynchronization) { + // Fail the `canActAsPrimary` check if the current leaseholder has + // not opted into multi-tab synchronization. If this happens at + // client startup, we reject the Promise returned by + // `enablePersistence()` and the user can continue to use Firestore + // with in-memory persistence. + // If this fails during a lease refresh, we will instead block the + // AsyncQueue from executing further operations. Note that this is + // acceptable since mixing & matching different `synchronizeTabs` + // settings is not supported. + // + // TODO(multitab): Remove this check when `synchronizeTabs` can no + // longer be turned off. + throw new FirestoreError( + Code.FAILED_PRECONDITION, + PRIMARY_LEASE_EXCLUSIVE_ERROR_MSG + ); + } + + return false; + } + } + + if (this.networkEnabled && this.inForeground) { + return true; + } + + let canActAsPrimary = true; + return clientMetadataStore(txn) + .iterate((key, otherClient, control) => { + if ( + this.clientId !== otherClient.clientId && + this.isWithinMaxAge(otherClient.updateTimeMs) && + !this.isClientZombied(otherClient.clientId) + ) { + const otherClientHasBetterNetworkState = + !this.networkEnabled && otherClient.networkEnabled; + const otherClientHasBetterVisibility = + !this.inForeground && otherClient.inForeground; + const otherClientHasSameNetworkState = + this.networkEnabled === otherClient.networkEnabled; + if ( + otherClientHasBetterNetworkState || + (otherClientHasBetterVisibility && + otherClientHasSameNetworkState) + ) { + canActAsPrimary = false; + control.done(); + } + } + }) + .next(() => canActAsPrimary); + }) + .next(canActAsPrimary => { + if (this.isPrimary !== canActAsPrimary) { + log.debug( + LOG_TAG, + `Client ${ + canActAsPrimary ? 'is' : 'is not' + } eligible for a primary lease.` + ); + } + return canActAsPrimary; + }); + } + + async shutdown(deleteData?: boolean): Promise { + // The shutdown() operations are idempotent and can be called even when + // start() aborted (e.g. because it couldn't acquire the persistence lease). this._started = false; + + this.markClientZombied(); + if (this.clientMetadataRefresher) { + this.clientMetadataRefresher.cancel(); + } + this.detachVisibilityHandler(); this.detachWindowUnloadHook(); - this.stopOwnerLeaseRefreshes(); - return this.releaseOwnerLease().then(() => { - this.simpleDb.close(); - if (deleteData) { - return SimpleDb.delete(this.dbName); + await this.simpleDb.runTransaction( + 'readwrite', + [DbOwner.store, DbClientMetadata.store], + txn => { + return this.releasePrimaryLeaseIfHeld(txn).next(() => + this.removeClientMetadata(txn) + ); } - }); + ); + this.simpleDb.close(); + + // Remove the entry marking the client as zombied from LocalStorage since + // we successfully deleted its metadata from IndexedDb. + this.removeClientZombiedEntry(); + if (deleteData) { + await SimpleDb.delete(this.dbName); + } + } + + getActiveClients(): Promise { + const clientIds: ClientId[] = []; + return this.simpleDb + .runTransaction('readonly', [DbClientMetadata.store], txn => { + return clientMetadataStore(txn).iterate((key, value) => { + if (this.isWithinMaxAge(value.updateTimeMs)) { + clientIds.push(value.clientId); + } + }); + }) + .then(() => clientIds); } get started(): boolean { @@ -180,21 +457,41 @@ export class IndexedDbPersistence implements Persistence { } getMutationQueue(user: User): MutationQueue { + assert( + this.started, + 'Cannot initialize MutationQueue before persistence is started.' + ); return IndexedDbMutationQueue.forUser(user, this.serializer); } getQueryCache(): QueryCache { + assert( + this.started, + 'Cannot initialize QueryCache before persistence is started.' + ); return new IndexedDbQueryCache(this.serializer); } getRemoteDocumentCache(): RemoteDocumentCache { - return new IndexedDbRemoteDocumentCache(this.serializer); + assert( + this.started, + 'Cannot initialize RemoteDocumentCache before persistence is started.' + ); + return new IndexedDbRemoteDocumentCache( + this.serializer, + /*keepDocumentChangeLog=*/ this.allowTabSynchronization + ); } runTransaction( action: string, - operation: (transaction: PersistenceTransaction) => PersistencePromise + requirePrimaryLease: boolean, + transactionOperation: ( + transaction: PersistenceTransaction + ) => PersistencePromise ): Promise { + // TODO(multitab): Consider removing `requirePrimaryLease` and exposing + // three different write modes (readonly, readwrite, readwrite_primary). if (this.persistenceError) { return Promise.reject(this.persistenceError); } @@ -207,12 +504,85 @@ export class IndexedDbPersistence implements Persistence { 'readwrite', ALL_STORES, simpleDbTxn => { - // Verify that we still have the owner lease as part of every transaction. - return this.ensureOwnerLease(simpleDbTxn).next(() => - operation(new IndexedDbTransaction(simpleDbTxn)) - ); + if (requirePrimaryLease) { + // While we merely verify that we have (or can acquire) the lease + // immediately, we wait to extend the primary lease until after + // executing transactionOperation(). This ensures that even if the + // transactionOperation takes a long time, we'll use a recent + // leaseTimestampMs in the extended (or newly acquired) lease. + return this.canActAsPrimary(simpleDbTxn) + .next(canActAsPrimary => { + if (!canActAsPrimary) { + // TODO(multitab): Handle this gracefully and transition back to + // secondary state. + log.error( + `Failed to obtain primary lease for action '${action}'.` + ); + this.isPrimary = false; + this.queue.enqueueAndForget(() => + this.primaryStateListener(false) + ); + throw new FirestoreError( + Code.FAILED_PRECONDITION, + PRIMARY_LEASE_LOST_ERROR_MSG + ); + } + return transactionOperation( + new IndexedDbTransaction(simpleDbTxn) + ); + }) + .next(result => { + return this.acquireOrExtendPrimaryLease(simpleDbTxn).next( + () => result + ); + }); + } else { + return this.verifyAllowTabSynchronization(simpleDbTxn).next(() => + transactionOperation(new IndexedDbTransaction(simpleDbTxn)) + ); + } + } + ); + } + + /** + * Verifies that the current tab is the primary leaseholder or alternatively + * that the leaseholder has opted into multi-tab synchronization. + */ + // TODO(multitab): Remove this check when `synchronizeTabs` can no longer be + // turned off. + private verifyAllowTabSynchronization( + txn: SimpleDbTransaction + ): PersistencePromise { + const store = ownerStore(txn); + return store.get('owner').next(currentPrimary => { + const currentLeaseIsValid = + currentPrimary !== null && + this.isWithinMaxAge(currentPrimary.leaseTimestampMs) && + !this.isClientZombied(currentPrimary.ownerId); + + if (currentLeaseIsValid && !this.isLocalClient(currentPrimary)) { + if (!currentPrimary.allowTabSynchronization) { + throw new FirestoreError( + Code.FAILED_PRECONDITION, + PRIMARY_LEASE_EXCLUSIVE_ERROR_MSG + ); + } } + }); + } + + /** + * Obtains or extends the new primary lease for the local client. This + * method does not verify that the client is eligible for this lease. + */ + private acquireOrExtendPrimaryLease(txn): PersistencePromise { + const newPrimary = new DbOwner( + this.clientId, + this.allowTabSynchronization, + Date.now() ); + return ownerStore(txn).put('owner', newPrimary); } static isAvailable(): boolean { @@ -239,140 +609,73 @@ export class IndexedDbPersistence implements Persistence { return 'firestore/' + databaseInfo.persistenceKey + '/' + database + '/'; } - /** - * Acquires the owner lease if there's no valid owner. Else returns a rejected - * promise. - */ - private tryAcquireOwnerLease(): Promise { - // NOTE: Don't use this.runTransaction, since it requires us to already - // have the lease. - return this.simpleDb.runTransaction('readwrite', [DbOwner.store], txn => { - const store = txn.store(DbOwner.store); - return store.get('owner').next(dbOwner => { - if (!this.validOwner(dbOwner)) { - const newDbOwner = new DbOwner(this.ownerId, Date.now()); - log.debug( - LOG_TAG, - 'No valid owner. Acquiring owner lease. Current owner:', - dbOwner, - 'New owner:', - newDbOwner - ); - return store.put('owner', newDbOwner); - } else { - log.debug( - LOG_TAG, - 'Valid owner already. Failing. Current owner:', - dbOwner - ); - this.persistenceError = new FirestoreError( - Code.FAILED_PRECONDITION, - EXISTING_OWNER_ERROR_MSG - ); - return PersistencePromise.reject(this.persistenceError); - } - }); - }); - } - - /** Checks the owner lease and deletes it if we are the current owner. */ - private releaseOwnerLease(): Promise { - // NOTE: Don't use this.runTransaction, since it requires us to already - // have the lease. - return this.simpleDb.runTransaction('readwrite', [DbOwner.store], txn => { - const store = txn.store(DbOwner.store); - return store.get('owner').next(dbOwner => { - if (dbOwner !== null && dbOwner.ownerId === this.ownerId) { - log.debug(LOG_TAG, 'Releasing owner lease.'); - return store.delete('owner'); - } else { - return PersistencePromise.resolve(); - } - }); - }); - } - - /** - * Checks the owner lease and returns a rejected promise if we are not the - * current owner. This should be included in every transaction to guard - * against losing the owner lease. - */ - private ensureOwnerLease(txn: SimpleDbTransaction): PersistencePromise { - const store = txn.store(DbOwner.store); - return store.get('owner').next(dbOwner => { - if (dbOwner === null || dbOwner.ownerId !== this.ownerId) { - this.persistenceError = new FirestoreError( - Code.FAILED_PRECONDITION, - EXISTING_OWNER_ERROR_MSG - ); - return PersistencePromise.reject(this.persistenceError); + /** Checks the primary lease and removes it if we are the current primary. */ + private releasePrimaryLeaseIfHeld( + txn: SimpleDbTransaction + ): PersistencePromise { + this.isPrimary = false; + + const store = ownerStore(txn); + return store.get('owner').next(primaryClient => { + if (this.isLocalClient(primaryClient)) { + log.debug(LOG_TAG, 'Releasing primary lease.'); + return store.delete('owner'); } else { return PersistencePromise.resolve(); } }); } - /** - * Returns true if the provided owner exists, has a recent timestamp, and - * isn't zombied. - * - * NOTE: To determine if the owner is zombied, this method reads from - * LocalStorage which could be mildly expensive. - */ - private validOwner(dbOwner: DbOwner | null): boolean { + /** Verifies that `updateTimeMs` is within CLIENT_STATE_MAX_AGE_MS. */ + private isWithinMaxAge(updateTimeMs: number): boolean { const now = Date.now(); - const minAcceptable = now - OWNER_LEASE_MAX_AGE_MS; + const minAcceptable = now - CLIENT_METADATA_MAX_AGE_MS; const maxAcceptable = now; - if (dbOwner === null) { - return false; // no owner. - } else if (dbOwner.leaseTimestampMs < minAcceptable) { - return false; // owner lease has expired. - } else if (dbOwner.leaseTimestampMs > maxAcceptable) { + if (updateTimeMs < minAcceptable) { + return false; + } else if (updateTimeMs > maxAcceptable) { log.error( - 'Persistence owner-lease is in the future. Discarding.', - dbOwner + `Detected an update time that is in the future: ${updateTimeMs} > ${maxAcceptable}` ); return false; - } else if (dbOwner.ownerId === this.getZombiedOwnerId()) { - return false; // owner's tab closed. - } else { - return true; } + + return true; } - /** - * Schedules a recurring timer to update the owner lease timestamp to prevent - * other tabs from taking the lease. - */ - private scheduleOwnerLeaseRefreshes(): void { - // NOTE: This doesn't need to be scheduled on the async queue and doing so - // would increase the chances of us not refreshing on time if the queue is - // backed up for some reason. - this.ownerLeaseRefreshHandle = setInterval(() => { - const txResult = this.simpleDb.runTransaction( - 'readwrite', - ALL_STORES, - txn => { - return this.ensureOwnerLease(txn).next(() => { - const store = txn.store(DbOwner.store); - return store.put('owner', new DbOwner(this.ownerId, Date.now())); - }); - } + private attachVisibilityHandler(): void { + if ( + this.document !== null && + typeof this.document.addEventListener === 'function' + ) { + this.documentVisibilityHandler = () => { + this.queue.enqueueAndForget(() => { + this.inForeground = this.document.visibilityState === 'visible'; + return this.updateClientMetadataAndTryBecomePrimary(); + }); + }; + + this.document.addEventListener( + 'visibilitychange', + this.documentVisibilityHandler ); - txResult.catch(reason => { - // Probably means we lost the lease. Report the error and stop trying to - // refresh the lease. - log.error(reason); - this.stopOwnerLeaseRefreshes(); - }); - }, OWNER_LEASE_REFRESH_INTERVAL_MS); + this.inForeground = this.document.visibilityState === 'visible'; + } } - private stopOwnerLeaseRefreshes(): void { - if (this.ownerLeaseRefreshHandle) { - clearInterval(this.ownerLeaseRefreshHandle); - this.ownerLeaseRefreshHandle = null; + private detachVisibilityHandler(): void { + if (this.documentVisibilityHandler) { + assert( + this.document !== null && + typeof this.document.addEventListener === 'function', + "Expected 'document.addEventListener' to be a function" + ); + this.document.removeEventListener( + 'visibilitychange', + this.documentVisibilityHandler + ); + this.documentVisibilityHandler = null; } } @@ -386,31 +689,30 @@ export class IndexedDbPersistence implements Persistence { * a synchronous API and so can be used reliably from an unload handler. */ private attachWindowUnloadHook(): void { - if ( - typeof window === 'object' && - typeof window.addEventListener === 'function' - ) { + if (typeof this.window.addEventListener === 'function') { this.windowUnloadHandler = () => { - // Record that we're zombied. - this.setZombiedOwnerId(this.ownerId); - - // Attempt graceful shutdown (including releasing our owner lease), but - // there's no guarantee it will complete. - // tslint:disable-next-line:no-floating-promises - this.shutdown(); + // Note: In theory, this should be scheduled on the AsyncQueue since it + // accesses internal state. We execute this code directly during shutdown + // to make sure it gets a chance to run. + this.markClientZombied(); + + this.queue.enqueueAndForget(() => { + // Attempt graceful shutdown (including releasing our owner lease), but + // there's no guarantee it will complete. + return this.shutdown(); + }); }; - window.addEventListener('unload', this.windowUnloadHandler); + this.window.addEventListener('unload', this.windowUnloadHandler); } } private detachWindowUnloadHook(): void { if (this.windowUnloadHandler) { assert( - typeof window === 'object' && - typeof window.removeEventListener === 'function', + typeof this.window.removeEventListener === 'function', "Expected 'window.removeEventListener' to be a function" ); - window.removeEventListener('unload', this.windowUnloadHandler); + this.window.removeEventListener('unload', this.windowUnloadHandler); this.windowUnloadHandler = null; } } @@ -420,46 +722,89 @@ export class IndexedDbPersistence implements Persistence { * zombied due to their tab closing) from LocalStorage, or null if no such * record exists. */ - private getZombiedOwnerId(): string | null { + private isClientZombied(clientId: ClientId): boolean { + if (this.window.localStorage === undefined) { + assert( + process.env.USE_MOCK_PERSISTENCE === 'YES', + 'Operating without LocalStorage is only supported with IndexedDbShim.' + ); + return null; + } + try { - const zombiedOwnerId = window.localStorage.getItem( - this.zombiedOwnerLocalStorageKey() + const isZombied = + this.window.localStorage.getItem( + this.zombiedClientLocalStorageKey(clientId) + ) !== null; + log.debug( + LOG_TAG, + `Client '${clientId}' ${ + isZombied ? 'is' : 'is not' + } zombied in LocalStorage` ); - log.debug(LOG_TAG, 'Zombied ownerID from LocalStorage:', zombiedOwnerId); - return zombiedOwnerId; + return isZombied; } catch (e) { // Gracefully handle if LocalStorage isn't available / working. - log.error('Failed to get zombie owner id.', e); + log.error(LOG_TAG, 'Failed to get zombied client id.', e); return null; } } /** - * Records a zombied owner (an owner that had its tab closed) in LocalStorage - * or, if passed null, deletes any recorded zombied owner. + * Record client as zombied (a client that had its tab closed). Zombied + * clients are ignored during primary tab selection. */ - private setZombiedOwnerId(zombieOwnerId: string | null): void { + private markClientZombied(): void { try { - if (zombieOwnerId === null) { - window.localStorage.removeItem(this.zombiedOwnerLocalStorageKey()); - } else { - window.localStorage.setItem( - this.zombiedOwnerLocalStorageKey(), - zombieOwnerId - ); - } + // TODO(multitab): Garbage Collect Local Storage + this.window.localStorage.setItem( + this.zombiedClientLocalStorageKey(this.clientId), + String(Date.now()) + ); } catch (e) { // Gracefully handle if LocalStorage isn't available / working. log.error('Failed to set zombie owner id.', e); } } - private zombiedOwnerLocalStorageKey(): string { - return this.localStoragePrefix + ZOMBIE_OWNER_LOCALSTORAGE_SUFFIX; + /** Removes the zombied client entry if it exists. */ + private removeClientZombiedEntry(): void { + try { + this.window.localStorage.removeItem( + this.zombiedClientLocalStorageKey(this.clientId) + ); + } catch (e) { + // Ignore + } } - private generateOwnerId(): string { - // For convenience, just use an AutoId. - return AutoId.newId(); + private zombiedClientLocalStorageKey(clientId: ClientId): string { + return `${ZOMBIED_CLIENTS_KEY_PREFIX}_${this.persistenceKey}_${clientId}`; } } + +export function isPrimaryLeaseLostError(err: FirestoreError): boolean { + return ( + err.code === Code.FAILED_PRECONDITION && + err.message === PRIMARY_LEASE_LOST_ERROR_MSG + ); +} +/** + * Helper to get a typed SimpleDbStore for the owner object store. + */ +function ownerStore( + txn: SimpleDbTransaction +): SimpleDbStore { + return txn.store(DbOwner.store); +} + +/** + * Helper to get a typed SimpleDbStore for the client metadata object store. + */ +function clientMetadataStore( + txn: SimpleDbTransaction +): SimpleDbStore { + return txn.store( + DbClientMetadata.store + ); +} diff --git a/packages/firestore/src/local/indexeddb_query_cache.ts b/packages/firestore/src/local/indexeddb_query_cache.ts index 2af34227d60..e268a7680a9 100644 --- a/packages/firestore/src/local/indexeddb_query_cache.ts +++ b/packages/firestore/src/local/indexeddb_query_cache.ts @@ -38,61 +38,68 @@ import { PersistenceTransaction } from './persistence'; import { PersistencePromise } from './persistence_promise'; import { QueryCache } from './query_cache'; import { QueryData } from './query_data'; +import { TargetIdGenerator } from '../core/target_id_generator'; import { SimpleDbStore } from './simple_db'; import { IndexedDbPersistence } from './indexeddb_persistence'; export class IndexedDbQueryCache implements QueryCache { constructor(private serializer: LocalSerializer) {} - /** - * The last received snapshot version. We store this separately from the - * metadata to avoid the extra conversion to/from DbTimestamp. - */ - private lastRemoteSnapshotVersion = SnapshotVersion.MIN; - - /** - * A cached copy of the metadata for the query cache. - */ - private metadata: DbTargetGlobal = null; - /** The garbage collector to notify about potential garbage keys. */ private garbageCollector: GarbageCollector | null = null; + // PORTING NOTE: We don't cache global metadata for the query cache, since + // some of it (in particular `highestTargetId`) can be modified by secondary + // tabs. We could perhaps be more granular (and e.g. still cache + // `lastRemoteSnapshotVersion` in memory) but for simplicity we currently go + // to IndexedDb whenever we need to read metadata. We can revisit if it turns + // out to have a meaningful performance impact. + + private targetIdGenerator = TargetIdGenerator.forQueryCache(); + start(transaction: PersistenceTransaction): PersistencePromise { - return globalTargetStore(transaction) - .get(DbTargetGlobal.key) - .next(metadata => { - assert( - metadata !== null, - 'Missing metadata row that should be added by schema migration.' - ); - this.metadata = metadata; - const lastSavedVersion = metadata.lastRemoteSnapshotVersion; - this.lastRemoteSnapshotVersion = SnapshotVersion.fromTimestamp( - new Timestamp(lastSavedVersion.seconds, lastSavedVersion.nanoseconds) - ); - return PersistencePromise.resolve(); - }); + // Nothing to do. + return PersistencePromise.resolve(); } - getHighestTargetId(): TargetId { - return this.metadata.highestTargetId; + allocateTargetId( + transaction: PersistenceTransaction + ): PersistencePromise { + return this.retrieveMetadata(transaction).next(metadata => { + metadata.highestTargetId = this.targetIdGenerator.after( + metadata.highestTargetId + ); + return this.saveMetadata(transaction, metadata).next( + () => metadata.highestTargetId + ); + }); } - getLastRemoteSnapshotVersion(): SnapshotVersion { - return this.lastRemoteSnapshotVersion; + getLastRemoteSnapshotVersion( + transaction: PersistenceTransaction + ): PersistencePromise { + return this.retrieveMetadata(transaction).next(metadata => { + return SnapshotVersion.fromTimestamp( + new Timestamp( + metadata.lastRemoteSnapshotVersion.seconds, + metadata.lastRemoteSnapshotVersion.nanoseconds + ) + ); + }); } - setLastRemoteSnapshotVersion( + setTargetsMetadata( transaction: PersistenceTransaction, - snapshotVersion: SnapshotVersion + highestListenSequenceNumber: number, + lastRemoteSnapshotVersion?: SnapshotVersion ): PersistencePromise { - this.lastRemoteSnapshotVersion = snapshotVersion; - this.metadata.lastRemoteSnapshotVersion = snapshotVersion.toTimestamp(); - return globalTargetStore(transaction).put( - DbTargetGlobal.key, - this.metadata - ); + return this.retrieveMetadata(transaction).next(metadata => { + metadata.highestListenSequenceNumber = highestListenSequenceNumber; + if (lastRemoteSnapshotVersion) { + metadata.lastRemoteSnapshotVersion = lastRemoteSnapshotVersion.toTimestamp(); + } + return this.saveMetadata(transaction, metadata); + }); } addQueryData( @@ -100,9 +107,11 @@ export class IndexedDbQueryCache implements QueryCache { queryData: QueryData ): PersistencePromise { return this.saveQueryData(transaction, queryData).next(() => { - this.metadata.targetCount += 1; - this.updateMetadataFromQueryData(queryData); - return this.saveMetadata(transaction); + return this.retrieveMetadata(transaction).next(metadata => { + metadata.targetCount += 1; + this.updateMetadataFromQueryData(queryData, metadata); + return this.saveMetadata(transaction, metadata); + }); }); } @@ -110,35 +119,39 @@ export class IndexedDbQueryCache implements QueryCache { transaction: PersistenceTransaction, queryData: QueryData ): PersistencePromise { - return this.saveQueryData(transaction, queryData).next(() => { - if (this.updateMetadataFromQueryData(queryData)) { - return this.saveMetadata(transaction); - } else { - return PersistencePromise.resolve(); - } - }); + return this.saveQueryData(transaction, queryData); } removeQueryData( transaction: PersistenceTransaction, queryData: QueryData ): PersistencePromise { - assert(this.metadata.targetCount > 0, 'Removing from an empty query cache'); return this.removeMatchingKeysForTargetId(transaction, queryData.targetId) .next(() => targetsStore(transaction).delete(queryData.targetId)) - .next(() => { - this.metadata.targetCount -= 1; - return this.saveMetadata(transaction); + .next(() => this.retrieveMetadata(transaction)) + .next(metadata => { + assert(metadata.targetCount > 0, 'Removing from an empty query cache'); + metadata.targetCount -= 1; + return this.saveMetadata(transaction, metadata); }); } - private saveMetadata( + private retrieveMetadata( transaction: PersistenceTransaction + ): PersistencePromise { + return globalTargetStore(transaction) + .get(DbTargetGlobal.key) + .next(metadata => { + assert(metadata !== null, 'Missing metadata row.'); + return metadata; + }); + } + + private saveMetadata( + transaction: PersistenceTransaction, + metadata: DbTargetGlobal ): PersistencePromise { - return globalTargetStore(transaction).put( - DbTargetGlobal.key, - this.metadata - ); + return globalTargetStore(transaction).put(DbTargetGlobal.key, metadata); } private saveQueryData( @@ -149,23 +162,29 @@ export class IndexedDbQueryCache implements QueryCache { } /** - * Updates the in-memory version of the metadata to account for values in the - * given QueryData. Saving is done separately. Returns true if there were any + * In-place updates the provided metadata to account for values in the given + * QueryData. Saving is done separately. Returns true if there were any * changes to the metadata. */ - private updateMetadataFromQueryData(queryData: QueryData): boolean { - let needsUpdate = false; - if (queryData.targetId > this.metadata.highestTargetId) { - this.metadata.highestTargetId = queryData.targetId; - needsUpdate = true; + private updateMetadataFromQueryData( + queryData: QueryData, + metadata: DbTargetGlobal + ): boolean { + if (queryData.targetId > metadata.highestTargetId) { + metadata.highestTargetId = queryData.targetId; + return true; } // TODO(GC): add sequence number check - return needsUpdate; + return false; } - get count(): number { - return this.metadata.targetCount; + getQueryCount( + transaction: PersistenceTransaction + ): PersistencePromise { + return this.retrieveMetadata(transaction).next( + metadata => metadata.targetCount + ); } getQueryData( @@ -330,6 +349,21 @@ export class IndexedDbQueryCache implements QueryCache { ) .next(() => count > 0); } + + getQueryDataForTarget( + transaction: PersistenceTransaction, + targetId: TargetId + ): PersistencePromise { + return targetsStore(transaction) + .get(targetId) + .next(found => { + if (found) { + return this.serializer.fromDbTarget(found); + } else { + return null; + } + }); + } } /** diff --git a/packages/firestore/src/local/indexeddb_remote_document_cache.ts b/packages/firestore/src/local/indexeddb_remote_document_cache.ts index da51ef0a8d9..53a4340c1b1 100644 --- a/packages/firestore/src/local/indexeddb_remote_document_cache.ts +++ b/packages/firestore/src/local/indexeddb_remote_document_cache.ts @@ -15,35 +15,99 @@ */ import { Query } from '../core/query'; -import { DocumentMap, documentMap } from '../model/collections'; -import { Document, MaybeDocument } from '../model/document'; +import { + documentKeySet, + DocumentMap, + documentMap, + MaybeDocumentMap, + maybeDocumentMap +} from '../model/collections'; +import { Document, MaybeDocument, NoDocument } from '../model/document'; import { DocumentKey } from '../model/document_key'; -import { DbRemoteDocument, DbRemoteDocumentKey } from './indexeddb_schema'; +import { + DbRemoteDocument, + DbRemoteDocumentKey, + DbRemoteDocumentChanges, + DbRemoteDocumentChangesKey +} from './indexeddb_schema'; import { IndexedDbPersistence } from './indexeddb_persistence'; import { LocalSerializer } from './local_serializer'; import { PersistenceTransaction } from './persistence'; import { PersistencePromise } from './persistence_promise'; import { RemoteDocumentCache } from './remote_document_cache'; +import { SnapshotVersion } from '../core/snapshot_version'; +import { assert } from '../util/assert'; import { SimpleDbStore } from './simple_db'; export class IndexedDbRemoteDocumentCache implements RemoteDocumentCache { - constructor(private serializer: LocalSerializer) {} + /** The last id read by `getNewDocumentChanges()`. */ + private lastReturnedDocumentChangesId = 0; - addEntry( + /** + * @param {LocalSerializer} serializer The document serializer. + * @param keepDocumentChangeLog Whether to keep a document change log in + * IndexedDb. This change log is required for Multi-Tab synchronization, but + * not needed in clients that don't share access to their remote document + * cache. + */ + constructor( + private readonly serializer: LocalSerializer, + private readonly keepDocumentChangeLog: boolean + ) {} + + start(transaction: PersistenceTransaction): PersistencePromise { + // If there are no existing changes, we set `lastReturnedDocumentChangesId` + // to 0 since IndexedDb's auto-generated keys start at 1. + this.lastReturnedDocumentChangesId = 0; + + return documentChangesStore(transaction).iterate( + { keysOnly: true, reverse: true }, + (key, value, control) => { + this.lastReturnedDocumentChangesId = key; + control.done(); + } + ); + } + + addEntries( transaction: PersistenceTransaction, - maybeDocument: MaybeDocument + maybeDocuments: MaybeDocument[] ): PersistencePromise { - return remoteDocumentsStore(transaction).put( - dbKey(maybeDocument.key), - this.serializer.toDbRemoteDocument(maybeDocument) - ); + const promises: Array> = []; + + if (maybeDocuments.length > 0) { + const documentStore = remoteDocumentsStore(transaction); + let changedKeys = documentKeySet(); + for (const maybeDocument of maybeDocuments) { + promises.push( + documentStore.put( + dbKey(maybeDocument.key), + this.serializer.toDbRemoteDocument(maybeDocument) + ) + ); + changedKeys = changedKeys.add(maybeDocument.key); + } + + if (this.keepDocumentChangeLog) { + // TODO(multitab): GC the documentChanges store. + promises.push( + documentChangesStore(transaction).put({ + changes: this.serializer.toDbResourcePaths(changedKeys) + }) + ); + } + } + + return PersistencePromise.waitFor(promises); } removeEntry( transaction: PersistenceTransaction, documentKey: DocumentKey ): PersistencePromise { + // We don't need to keep changelog for these removals since `removeEntry` is + // only used for garbage collection. return remoteDocumentsStore(transaction).delete(dbKey(documentKey)); } @@ -81,6 +145,45 @@ export class IndexedDbRemoteDocumentCache implements RemoteDocumentCache { }) .next(() => results); } + + getNewDocumentChanges( + transaction: PersistenceTransaction + ): PersistencePromise { + assert( + this.keepDocumentChangeLog, + 'Can only call getNewDocumentChanges() when document change log is enabled' + ); + let changedKeys = documentKeySet(); + let changedDocs = maybeDocumentMap(); + + const range = IDBKeyRange.lowerBound( + this.lastReturnedDocumentChangesId, + /*lowerOpen=*/ true + ); + + return documentChangesStore(transaction) + .iterate({ range }, (_, documentChange) => { + changedKeys = changedKeys.unionWith( + this.serializer.fromDbResourcePaths(documentChange.changes) + ); + this.lastReturnedDocumentChangesId = documentChange.id; + }) + .next(() => { + const documentPromises: Array> = []; + changedKeys.forEach(key => { + documentPromises.push( + this.getEntry(transaction, key).next(maybeDoc => { + changedDocs = changedDocs.insert( + key, + maybeDoc || new NoDocument(key, SnapshotVersion.forDeletedDoc()) + ); + }) + ); + }); + return PersistencePromise.waitFor(documentPromises); + }) + .next(() => changedDocs); + } } /** @@ -95,6 +198,19 @@ function remoteDocumentsStore( ); } +/** + * Helper to get a typed SimpleDbStore for the remoteDocumentChanges object + * store. + */ +function documentChangesStore( + txn: PersistenceTransaction +): SimpleDbStore { + return IndexedDbPersistence.getStore< + DbRemoteDocumentChangesKey, + DbRemoteDocumentChanges + >(txn, DbRemoteDocumentChanges.store); +} + function dbKey(docKey: DocumentKey): DbRemoteDocumentKey { return docKey.path.toArray(); } diff --git a/packages/firestore/src/local/indexeddb_schema.ts b/packages/firestore/src/local/indexeddb_schema.ts index b8d1332a3bf..a1ef19bea44 100644 --- a/packages/firestore/src/local/indexeddb_schema.ts +++ b/packages/firestore/src/local/indexeddb_schema.ts @@ -34,8 +34,9 @@ import { SnapshotVersion } from '../core/snapshot_version'; * 3. Dropped and re-created Query Cache to deal with cache corruption related * to limbo resolution. Addresses * https://github.com/firebase/firebase-ios-sdk/issues/1548 + * 4. Multi-Tab Support. */ -export const SCHEMA_VERSION = 3; +export const SCHEMA_VERSION = 4; /** * Performs database creation and schema upgrades. @@ -76,6 +77,23 @@ export function createOrUpgradeDb( p = p.next(() => writeEmptyTargetGlobalEntry(txn)); } + if (fromVersion < 4 && toVersion >= 4) { + if (fromVersion !== 0) { + // Schema version 3 uses auto-generated keys to generate globally unique + // mutation batch IDs (this was previously ensured internally by the + // client). To migrate to the new schema, we have to read all mutations + // and write them back out. We preserve the existing batch IDs to guarantee + // consistency with other object stores. Any further mutation batch IDs will + // be auto-generated. + p = p.next(() => upgradeMutationBatchSchemaAndMigrateData(db, txn)); + } + + p = p.next(() => { + createClientMetadataStore(db); + createRemoteDocumentChangesStore(db); + }); + } + return p; } @@ -101,11 +119,18 @@ export type DbOwnerKey = 'owner'; * should regularly write an updated timestamp to prevent other tabs from * "stealing" ownership of the db. */ +// TODO(multitab): Rename this class to reflect the primary/secondary naming +// in the rest of the client. export class DbOwner { /** Name of the IndexedDb object store. */ static store = 'owner'; - constructor(public ownerId: string, public leaseTimestampMs: number) {} + constructor( + public ownerId: string, + /** Whether to allow shared access from multiple tabs. */ + public allowTabSynchronization: boolean, + public leaseTimestampMs: number + ) {} } function createOwnerStore(db: IDBDatabase): void { @@ -153,8 +178,8 @@ export class DbMutationQueue { ) {} } -/** keys in the 'mutations' object store are [userId, batchId] pairs. */ -export type DbMutationBatchKey = [string, BatchId]; +/** The 'mutations' store is keyed by batch ID. */ +export type DbMutationBatchKey = BatchId; /** * An object to be stored in the 'mutations' store in IndexedDb. @@ -168,7 +193,13 @@ export class DbMutationBatch { static store = 'mutations'; /** Keys are automatically assigned via the userId, batchId properties. */ - static keyPath = ['userId', 'batchId']; + static keyPath = 'batchId'; + + /** The index name for lookup of mutations by user. */ + static userMutationsIndex = 'userMutationsIndex'; + + /** The user mutations index is keyed by [userId, batchId] pairs. */ + static userMutationsKeyPath = ['userId', 'batchId']; constructor( /** @@ -176,8 +207,7 @@ export class DbMutationBatch { */ public userId: string, /** - * An identifier for this batch, allocated by the mutation queue in a - * monotonically increasing manner. + * An identifier for this batch, allocated using an auto-generated key. */ public batchId: BatchId, /** @@ -206,13 +236,54 @@ function createMutationQueue(db: IDBDatabase): void { keyPath: DbMutationQueue.keyPath }); - db.createObjectStore(DbMutationBatch.store, { - keyPath: DbMutationBatch.keyPath as KeyPath + const mutationBatchesStore = db.createObjectStore(DbMutationBatch.store, { + keyPath: DbMutationBatch.keyPath, + autoIncrement: true }); + mutationBatchesStore.createIndex( + DbMutationBatch.userMutationsIndex, + DbMutationBatch.userMutationsKeyPath, + { unique: true } + ); db.createObjectStore(DbDocumentMutation.store); } +/** + * Upgrade function to migrate the 'mutations' store from V1 to V3. Loads + * and rewrites all data. + */ +function upgradeMutationBatchSchemaAndMigrateData( + db: IDBDatabase, + txn: SimpleDbTransaction +): PersistencePromise { + const v1MutationsStore = txn.store<[string, number], DbMutationBatch>( + DbMutationBatch.store + ); + return v1MutationsStore.loadAll().next(existingMutations => { + db.deleteObjectStore(DbMutationBatch.store); + + const mutationsStore = db.createObjectStore(DbMutationBatch.store, { + keyPath: DbMutationBatch.keyPath, + autoIncrement: true + }); + mutationsStore.createIndex( + DbMutationBatch.userMutationsIndex, + DbMutationBatch.userMutationsKeyPath, + { unique: true } + ); + + const v3MutationsStore = txn.store( + DbMutationBatch.store + ); + const writeAll = existingMutations.map(mutation => + v3MutationsStore.put(mutation) + ); + + return PersistencePromise.waitFor(writeAll); + }); +} + /** * An object to be stored in the 'documentMutations' store in IndexedDb. * @@ -543,11 +614,77 @@ function writeEmptyTargetGlobalEntry( } /** - * The list of all default IndexedDB stores used throughout the SDK. This is - * used when creating transactions so that access across all stores is done - * atomically. + * An object store to store the keys of changed documents. This is used to + * facilitate storing document changelogs in the Remote Document Cache. + * + * PORTING NOTE: This is used for change propagation during multi-tab syncing + * and not needed on iOS and Android. + */ +export class DbRemoteDocumentChanges { + /** Name of the IndexedDb object store. */ + static store = 'remoteDocumentChanges'; + + /** Keys are auto-generated via the `id` property. */ + static keyPath = 'id'; + + /** The auto-generated key of this entry. */ + id?: number; + + constructor( + /** The keys of the changed documents. */ + public changes: EncodedResourcePath[] + ) {} +} + +/* + * The key for DbRemoteDocumentChanges, consisting of an auto-incrementing + * number. +*/ +export type DbRemoteDocumentChangesKey = number; + +function createRemoteDocumentChangesStore(db: IDBDatabase): void { + db.createObjectStore(DbRemoteDocumentChanges.store, { + keyPath: 'id', + autoIncrement: true + }); +} + +/** + * A record of the metadata state of each client. + * + * PORTING NOTE: This is used to synchronize multi-tab state and does not need + * to be ported to iOS or Android. */ -export const ALL_STORES = [ +export class DbClientMetadata { + /** Name of the IndexedDb object store. */ + static store = 'clientMetadata'; + + /** Keys are automatically assigned via the clientId properties. */ + static keyPath = 'clientId'; + + constructor( + /** The auto-generated client id assigned at client startup. */ + public clientId: string, + /** The last time this state was updated. */ + public updateTimeMs: number, + /** Whether the client's network connection is enabled. */ + public networkEnabled: boolean, + /** Whether this client is running in a foreground tab. */ + public inForeground: boolean + ) {} +} + +/** Object keys in the 'clientMetadata' store are clientId strings. */ +export type DbClientMetadataKey = string; + +function createClientMetadataStore(db: IDBDatabase): void { + db.createObjectStore(DbClientMetadata.store, { + keyPath: DbClientMetadata.keyPath + }); +} + +// Visible for testing +export const V1_STORES = [ DbMutationQueue.store, DbMutationBatch.store, DbDocumentMutation.store, @@ -557,3 +694,22 @@ export const ALL_STORES = [ DbTargetGlobal.store, DbTargetDocument.store ]; + +// V2 is no longer usable (see comment at top of file) + +// Visible for testing +export const V3_STORES = V1_STORES; + +// Visible for testing +export const V4_STORES = [ + ...V3_STORES, + DbClientMetadata.store, + DbRemoteDocumentChanges.store +]; + +/** + * The list of all default IndexedDB stores used throughout the SDK. This is + * used when creating transactions so that access across all stores is done + * atomically. + */ +export const ALL_STORES = V4_STORES; diff --git a/packages/firestore/src/local/local_documents_view.ts b/packages/firestore/src/local/local_documents_view.ts index 7c71a34c9ab..78f33d9e050 100644 --- a/packages/firestore/src/local/local_documents_view.ts +++ b/packages/firestore/src/local/local_documents_view.ts @@ -110,7 +110,11 @@ export class LocalDocumentsView { }); } - /** Performs a query against the local view of all documents. */ + /** + * Performs a query against the local view of all documents. + * + * Multi-Tab Note: This operation is safe to use from secondary clients. + */ getDocumentsMatchingQuery( transaction: PersistenceTransaction, query: Query diff --git a/packages/firestore/src/local/local_serializer.ts b/packages/firestore/src/local/local_serializer.ts index 5fa4d85045f..211d051a02e 100644 --- a/packages/firestore/src/local/local_serializer.ts +++ b/packages/firestore/src/local/local_serializer.ts @@ -33,6 +33,8 @@ import { DbTimestamp } from './indexeddb_schema'; import { QueryData, QueryPurpose } from './query_data'; +import { decode, encode, EncodedResourcePath } from './encoded_resource_path'; +import { documentKeySet, DocumentKeySet } from '../model/collections'; /** Serializer for values stored in the LocalStore. */ export class LocalSerializer { @@ -90,6 +92,30 @@ export class LocalSerializer { return new MutationBatch(dbBatch.batchId, timestamp, mutations); } + /* + * Encodes a set of document keys into an array of EncodedResourcePaths. + */ + toDbResourcePaths(keys: DocumentKeySet): EncodedResourcePath[] { + const encodedKeys: EncodedResourcePath[] = []; + + keys.forEach(key => { + encodedKeys.push(encode(key.path)); + }); + + return encodedKeys; + } + + /** Decodes an array of EncodedResourcePaths into a set of document keys. */ + fromDbResourcePaths(encodedPaths: EncodedResourcePath[]): DocumentKeySet { + let keys = documentKeySet(); + + for (const documentKey of encodedPaths) { + keys = keys.add(new DocumentKey(decode(documentKey))); + } + + return keys; + } + /** Decodes a DbTarget into QueryData */ fromDbTarget(dbTarget: DbTarget): QueryData { const readTime = new Timestamp( diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 5b0b797bb5d..b258d9eb4ac 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -18,7 +18,6 @@ import { Timestamp } from '../api/timestamp'; import { User } from '../auth/user'; import { Query } from '../core/query'; import { SnapshotVersion } from '../core/snapshot_version'; -import { TargetIdGenerator } from '../core/target_id_generator'; import { BatchId, ProtoByteString, TargetId } from '../core/types'; import { DocumentKeySet, @@ -50,6 +49,7 @@ import { QueryData, QueryPurpose } from './query_data'; import { ReferenceSet } from './reference_set'; import { RemoteDocumentCache } from './remote_document_cache'; import { RemoteDocumentChangeBuffer } from './remote_document_change_buffer'; +import { ClientId, SharedClientState } from './shared_client_state'; const LOG_TAG = 'LocalStore'; @@ -59,6 +59,13 @@ export interface LocalWriteResult { changes: MaybeDocumentMap; } +/** The result of a user-change operation in the local store. */ +export interface UserChangeResult { + readonly affectedDocuments: MaybeDocumentMap; + readonly removedBatchIds: BatchId[]; + readonly addedBatchIds: BatchId[]; +} + /** * Local storage in the Firestore client. Coordinates persistence components * like the mutation queue and remote document cache to present a @@ -144,11 +151,9 @@ export class LocalStore { private queryCache: QueryCache; /** Maps a targetID to data about its query. */ + // TODO(multitab): Rename to `queryDataByTarget` to match SyncEngine's naming. private targetIds = {} as { [targetId: number]: QueryData }; - /** Used to generate targetIDs for queries tracked locally. */ - private targetIdGenerator = TargetIdGenerator.forLocalStore(); - /** * A heldBatchResult is a mutation batch result (from a write acknowledgement) * that arrived before the watch stream got notified of a snapshot that @@ -170,7 +175,15 @@ export class LocalStore { * cached (e.g. if they are no longer retained by the above reference sets * and the garbage collector is performing eager collection). */ - private garbageCollector: GarbageCollector + private garbageCollector: GarbageCollector, + /** + * SharedClientState to notify of acknowledged writes. + * + * TODO(mrschmidt): When we get rid of held write acks, the SyncEngine can + * notify SharedClientState of all write acknowledgements and LocalStore + * should no longer need access to SharedClientState. + */ + private sharedClientState: SharedClientState ) { assert( persistence.started, @@ -190,8 +203,11 @@ export class LocalStore { /** Performs any initial startup actions required by the local store. */ start(): Promise { - return this.persistence.runTransaction('Start LocalStore', txn => { - return this.startMutationQueue(txn).next(() => this.startQueryCache(txn)); + // TODO(multitab): Ensure that we in fact don't need the primary lease. + return this.persistence.runTransaction('Start LocalStore', false, txn => { + return this.startMutationQueue(txn) + .next(() => this.startQueryCache(txn)) + .next(() => this.startRemoteDocumentCache(txn)); }); } @@ -201,8 +217,8 @@ export class LocalStore { * In response the local store switches the mutation queue to the new user and * returns any resulting document changes. */ - handleUserChange(user: User): Promise { - return this.persistence.runTransaction('Handle user change', txn => { + handleUserChange(user: User): Promise { + return this.persistence.runTransaction('Handle user change', false, txn => { // Swap out the mutation queue, grabbing the pending mutation batches // before and after. let oldBatches: MutationBatch[]; @@ -226,19 +242,37 @@ export class LocalStore { return this.mutationQueue.getAllMutationBatches(txn); }) .next(newBatches => { + const removedBatchIds: BatchId[] = []; + const addedBatchIds: BatchId[] = []; + // Union the old/new changed keys. let changedKeys = documentKeySet(); - for (const batches of [oldBatches, newBatches]) { - for (const batch of batches) { - for (const mutation of batch.mutations) { - changedKeys = changedKeys.add(mutation.key); - } + + for (const batch of oldBatches) { + removedBatchIds.push(batch.batchId); + for (const mutation of batch.mutations) { + changedKeys = changedKeys.add(mutation.key); + } + } + + for (const batch of newBatches) { + addedBatchIds.push(batch.batchId); + for (const mutation of batch.mutations) { + changedKeys = changedKeys.add(mutation.key); } } - // Return the set of all (potentially) changed documents as the - // result of the user change. - return this.localDocuments.getDocuments(txn, changedKeys); + // Return the set of all (potentially) changed documents and the list + // of mutation batch IDs that were affected by change. + return this.localDocuments + .getDocuments(txn, changedKeys) + .next(affectedDocuments => { + return { + affectedDocuments, + removedBatchIds, + addedBatchIds + }; + }); }); }); } @@ -246,10 +280,7 @@ export class LocalStore { private startQueryCache( txn: PersistenceTransaction ): PersistencePromise { - return this.queryCache.start(txn).next(() => { - const targetId = this.queryCache.getHighestTargetId(); - this.targetIdGenerator = TargetIdGenerator.forLocalStore(targetId); - }); + return this.queryCache.start(txn); } private startMutationQueue( @@ -288,25 +319,55 @@ export class LocalStore { }); } + private startRemoteDocumentCache( + txn: PersistenceTransaction + ): PersistencePromise { + return this.remoteDocuments.start(txn); + } + /* Accept locally generated Mutations and commit them to storage. */ localWrite(mutations: Mutation[]): Promise { - return this.persistence.runTransaction('Locally write mutations', txn => { - let batch: MutationBatch; - const localWriteTime = Timestamp.now(); - return this.mutationQueue - .addMutationBatch(txn, localWriteTime, mutations) - .next(promisedBatch => { - batch = promisedBatch; - // TODO(koss): This is doing an N^2 update by replaying ALL the - // mutations on each document (instead of just the ones added) in - // this batch. - const keys = batch.keys(); - return this.localDocuments.getDocuments(txn, keys); - }) - .next((changedDocuments: MaybeDocumentMap) => { - return { batchId: batch.batchId, changes: changedDocuments }; - }); - }); + return this.persistence.runTransaction( + 'Locally write mutations', + false, + txn => { + let batch: MutationBatch; + const localWriteTime = Timestamp.now(); + return this.mutationQueue + .addMutationBatch(txn, localWriteTime, mutations) + .next(promisedBatch => { + batch = promisedBatch; + // TODO(koss): This is doing an N^2 update by replaying ALL the + // mutations on each document (instead of just the ones added) in + // this batch. + const keys = batch.keys(); + return this.localDocuments.getDocuments(txn, keys); + }) + .next((changedDocuments: MaybeDocumentMap) => { + return { batchId: batch.batchId, changes: changedDocuments }; + }); + } + ); + } + + /** Returns the local view of the documents affected by a mutation batch. */ + // PORTING NOTE: Multi-tab only. + lookupMutationDocuments(batchId: BatchId): Promise { + return this.persistence.runTransaction( + 'Lookup mutation documents', + false, + txn => { + return this.mutationQueue + .lookupMutationKeys(txn, batchId) + .next(keys => { + if (keys) { + return this.localDocuments.getDocuments(txn, keys); + } else { + return PersistencePromise.resolve(null); + } + }); + } + ); } /** @@ -326,12 +387,13 @@ export class LocalStore { acknowledgeBatch( batchResult: MutationBatchResult ): Promise { - return this.persistence.runTransaction('Acknowledge batch', txn => { + return this.persistence.runTransaction('Acknowledge batch', true, txn => { let affected: DocumentKeySet; return this.mutationQueue .acknowledgeBatch(txn, batchResult.batch, batchResult.streamToken) - .next(() => { - if (this.shouldHoldBatchResult(batchResult.commitVersion)) { + .next(() => this.shouldHoldBatchResult(txn, batchResult.commitVersion)) + .next(shouldHoldBatchResult => { + if (shouldHoldBatchResult) { this.heldBatchResults.push(batchResult); affected = documentKeySet(); return PersistencePromise.resolve(); @@ -365,7 +427,7 @@ export class LocalStore { * @returns The resulting modified documents. */ rejectBatch(batchId: BatchId): Promise { - return this.persistence.runTransaction('Reject batch', txn => { + return this.persistence.runTransaction('Reject batch', true, txn => { let toReject: MutationBatch; let affectedKeys: DocumentKeySet; return this.mutationQueue @@ -402,9 +464,13 @@ export class LocalStore { /** Returns the last recorded stream token for the current user. */ getLastStreamToken(): Promise { - return this.persistence.runTransaction('Get last stream token', txn => { - return this.mutationQueue.getLastStreamToken(txn); - }); + return this.persistence.runTransaction( + 'Get last stream token', + false, // TODO(multitab): This requires the owner lease + txn => { + return this.mutationQueue.getLastStreamToken(txn); + } + ); } /** @@ -413,17 +479,25 @@ export class LocalStore { * response to an error that requires clearing the stream token. */ setLastStreamToken(streamToken: ProtoByteString): Promise { - return this.persistence.runTransaction('Set last stream token', txn => { - return this.mutationQueue.setLastStreamToken(txn, streamToken); - }); + return this.persistence.runTransaction( + 'Set last stream token', + true, + txn => { + return this.mutationQueue.setLastStreamToken(txn, streamToken); + } + ); } /** * Returns the last consistent snapshot processed (used by the RemoteStore to * determine whether to buffer incoming snapshots from the backend). */ - getLastRemoteSnapshotVersion(): SnapshotVersion { - return this.queryCache.getLastRemoteSnapshotVersion(); + getLastRemoteSnapshotVersion(): Promise { + return this.persistence.runTransaction( + 'Get last remote snapshot version', + false, + txn => this.queryCache.getLastRemoteSnapshotVersion(txn) + ); } /** @@ -436,7 +510,7 @@ export class LocalStore { */ applyRemoteEvent(remoteEvent: RemoteEvent): Promise { const documentBuffer = new RemoteDocumentChangeBuffer(this.remoteDocuments); - return this.persistence.runTransaction('Apply remote event', txn => { + return this.persistence.runTransaction('Apply remote event', true, txn => { const promises = [] as Array>; let authoritativeUpdates = documentKeySet(); objUtils.forEachNumber( @@ -534,19 +608,25 @@ export class LocalStore { // can synthesize remote events when we get permission denied errors while // trying to resolve the state of a locally cached document that is in // limbo. - const lastRemoteVersion = this.queryCache.getLastRemoteSnapshotVersion(); const remoteVersion = remoteEvent.snapshotVersion; if (!remoteVersion.isEqual(SnapshotVersion.MIN)) { - assert( - remoteVersion.compareTo(lastRemoteVersion) >= 0, - 'Watch stream reverted to previous snapshot?? ' + - remoteVersion + - ' < ' + - lastRemoteVersion - ); - promises.push( - this.queryCache.setLastRemoteSnapshotVersion(txn, remoteVersion) - ); + const updateRemoteVersion = this.queryCache + .getLastRemoteSnapshotVersion(txn) + .next(lastRemoteVersion => { + assert( + remoteVersion.compareTo(lastRemoteVersion) >= 0, + 'Watch stream reverted to previous snapshot?? ' + + remoteVersion + + ' < ' + + lastRemoteVersion + ); + return this.queryCache.setTargetsMetadata( + txn, + /*highestSequenceNumber=*/ 0, + remoteVersion + ); + }); + promises.push(updateRemoteVersion); } let releasedWriteKeys: DocumentKeySet; @@ -632,15 +712,19 @@ export class LocalStore { * @returns The next mutation or null if there wasn't one. */ nextMutationBatch(afterBatchId?: BatchId): Promise { - return this.persistence.runTransaction('Get next mutation batch', txn => { - if (afterBatchId === undefined) { - afterBatchId = BATCHID_UNKNOWN; + return this.persistence.runTransaction( + 'Get next mutation batch', + false, + txn => { + if (afterBatchId === undefined) { + afterBatchId = BATCHID_UNKNOWN; + } + return this.mutationQueue.getNextMutationBatchAfterBatchId( + txn, + afterBatchId + ); } - return this.mutationQueue.getNextMutationBatchAfterBatchId( - txn, - afterBatchId - ); - }); + ); } /** @@ -648,7 +732,7 @@ export class LocalStore { * found - used for testing. */ readDocument(key: DocumentKey): Promise { - return this.persistence.runTransaction('read document', txn => { + return this.persistence.runTransaction('read document', false, txn => { return this.localDocuments.getDocument(txn, key); }); } @@ -659,7 +743,7 @@ export class LocalStore { * the store can be used to manage its view. */ allocateQuery(query: Query): Promise { - return this.persistence.runTransaction('Allocate query', txn => { + return this.persistence.runTransaction('Allocate query', false, txn => { let queryData: QueryData; return this.queryCache .getQueryData(txn, query) @@ -671,9 +755,10 @@ export class LocalStore { queryData = cached; return PersistencePromise.resolve(); } else { - const targetId = this.targetIdGenerator.next(); - queryData = new QueryData(query, targetId, QueryPurpose.Listen); - return this.queryCache.addQueryData(txn, queryData); + return this.queryCache.allocateTargetId(txn).next(targetId => { + queryData = new QueryData(query, targetId, QueryPurpose.Listen); + return this.queryCache.addQueryData(txn, queryData); + }); } }) .next(() => { @@ -687,53 +772,58 @@ export class LocalStore { }); } - /** Unpin all the documents associated with the given query. */ - releaseQuery(query: Query): Promise { - return this.persistence.runTransaction('Release query', txn => { - return this.queryCache - .getQueryData(txn, query) - .next((queryData: QueryData | null) => { - assert( - queryData != null, - 'Tried to release nonexistent query: ' + query - ); - - const targetId = queryData!.targetId; - const cachedQueryData = this.targetIds[targetId]; - - this.localViewReferences.removeReferencesForId(targetId); - delete this.targetIds[targetId]; - if (this.garbageCollector.isEager) { - return this.queryCache.removeQueryData(txn, queryData!); - } else if ( - cachedQueryData.snapshotVersion > queryData!.snapshotVersion - ) { - // If we've been avoiding persisting the resumeToken (see - // shouldPersistQueryData for conditions and rationale) we need to - // persist the token now because there will no longer be an - // in-memory version to fall back on. - return this.queryCache.updateQueryData(txn, cachedQueryData); - } else { - return PersistencePromise.resolve(); - } - }) - .next(() => { - // If this was the last watch target, then we won't get any more - // watch snapshots, so we should release any held batch results. - if (objUtils.isEmpty(this.targetIds)) { - const documentBuffer = new RemoteDocumentChangeBuffer( - this.remoteDocuments - ); - return this.releaseHeldBatchResults(txn, documentBuffer).next( - () => { - documentBuffer.apply(txn); - } + /** + * Unpin all the documents associated with the given query. If + * `keepPersistedQueryData` is set to false and Eager GC enabled, the method + * directly removes the associated query data from the query cache. + */ + // PORTING NOTE: `keepPersistedQueryData` is multi-tab only. + releaseQuery(query: Query, keepPersistedQueryData: boolean): Promise { + const requirePrimaryLease = keepPersistedQueryData === false; + return this.persistence.runTransaction( + 'Release query', + requirePrimaryLease, + txn => { + return this.queryCache + .getQueryData(txn, query) + .next((queryData: QueryData | null) => { + assert( + queryData != null, + 'Tried to release nonexistent query: ' + query ); - } else { - return PersistencePromise.resolve(); - } - }); - }); + const targetId = queryData!.targetId; + const cachedQueryData = this.targetIds[targetId]; + + this.localViewReferences.removeReferencesForId(targetId); + delete this.targetIds[targetId]; + if (!keepPersistedQueryData && this.garbageCollector.isEager) { + return this.queryCache.removeQueryData(txn, queryData!); + } else if ( + cachedQueryData.snapshotVersion > queryData!.snapshotVersion + ) { + // If we've been avoiding persisting the resumeToken (see + // shouldPersistQueryData for conditions and rationale) we need to + // persist the token now because there will no longer be an + // in-memory version to fall back on. + return this.queryCache.updateQueryData(txn, cachedQueryData); + } else { + return PersistencePromise.resolve(); + } + }) + .next(() => { + // If this was the last watch target, then we won't get any more + // watch snapshots, so we should release any held batch results. + if (objUtils.isEmpty(this.targetIds)) { + const documentBuffer = new RemoteDocumentChangeBuffer( + this.remoteDocuments + ); + return this.releaseHeldBatchResults(txn, documentBuffer).next( + () => documentBuffer.apply(txn) + ); + } + }); + } + ); } /** @@ -741,7 +831,7 @@ export class LocalStore { * returns the results. */ executeQuery(query: Query): Promise { - return this.persistence.runTransaction('Execute query', txn => { + return this.persistence.runTransaction('Execute query', false, txn => { return this.localDocuments.getDocumentsMatchingQuery(txn, query); }); } @@ -751,9 +841,13 @@ export class LocalStore { * target id in the remote table. */ remoteDocumentKeys(targetId: TargetId): Promise { - return this.persistence.runTransaction('Remote document keys', txn => { - return this.queryCache.getMatchingKeysForTargetId(txn, targetId); - }); + return this.persistence.runTransaction( + 'Remote document keys', + false, + txn => { + return this.queryCache.getMatchingKeysForTargetId(txn, targetId); + } + ); } /** @@ -765,7 +859,7 @@ export class LocalStore { collectGarbage(): Promise { // Call collectGarbage regardless of whether isGCEnabled so the referenceSet // doesn't continue to accumulate the garbage keys. - return this.persistence.runTransaction('Garbage collection', txn => { + return this.persistence.runTransaction('Garbage collection', true, txn => { return this.garbageCollector.collectGarbage(txn).next(garbage => { const promises = [] as Array>; garbage.forEach(key => { @@ -776,40 +870,82 @@ export class LocalStore { }); } + // PORTING NOTE: Multi-tab only. + getActiveClients(): Promise { + return this.persistence.getActiveClients(); + } + + // PORTING NOTE: Multi-tab only. + removeCachedMutationBatchMetadata(batchId: BatchId): void { + this.mutationQueue.removeCachedMutationKeys(batchId); + } + + // PORTING NOTE: Multi-tab only. + setNetworkEnabled(networkEnabled: boolean): void { + this.persistence.setNetworkEnabled(networkEnabled); + } + private releaseHeldBatchResults( txn: PersistenceTransaction, documentBuffer: RemoteDocumentChangeBuffer ): PersistencePromise { - const toRelease: MutationBatchResult[] = []; - for (const batchResult of this.heldBatchResults) { - if (!this.isRemoteUpToVersion(batchResult.commitVersion)) { - break; - } - toRelease.push(batchResult); - } + let writesToRelease: PersistencePromise; - if (toRelease.length === 0) { - return PersistencePromise.resolve(documentKeySet()); + if (objUtils.isEmpty(this.targetIds)) { + // We always release all writes when there are no active watch targets. + writesToRelease = PersistencePromise.resolve( + this.heldBatchResults.slice() + ); } else { - this.heldBatchResults.splice(0, toRelease.length); - return this.releaseBatchResults(txn, toRelease, documentBuffer); + writesToRelease = this.queryCache + .getLastRemoteSnapshotVersion(txn) + .next(lastRemoteVersion => { + const toRelease = []; + for (const batchResult of this.heldBatchResults) { + if (batchResult.commitVersion.compareTo(lastRemoteVersion) > 0) { + break; + } + toRelease.push(batchResult); + } + return toRelease; + }); } + + return writesToRelease.next(toRelease => { + if (toRelease.length === 0) { + return PersistencePromise.resolve(documentKeySet()); + } else { + this.heldBatchResults.splice(0, toRelease.length); + return this.releaseBatchResults(txn, toRelease, documentBuffer); + } + }); } - private isRemoteUpToVersion(version: SnapshotVersion): boolean { - // If there are no watch targets, then we won't get remote snapshots, and - // we are always "up-to-date." - const lastRemoteVersion = this.queryCache.getLastRemoteSnapshotVersion(); - return ( - version.compareTo(lastRemoteVersion) <= 0 || - objUtils.isEmpty(this.targetIds) - ); + private isRemoteUpToVersion( + txn: PersistenceTransaction, + version: SnapshotVersion + ): PersistencePromise { + return this.queryCache + .getLastRemoteSnapshotVersion(txn) + .next(lastRemoteVersion => { + return ( + version.compareTo(lastRemoteVersion) <= 0 || + objUtils.isEmpty(this.targetIds) + ); + }); } - private shouldHoldBatchResult(version: SnapshotVersion): boolean { + private shouldHoldBatchResult( + txn: PersistenceTransaction, + batchVersion: SnapshotVersion + ): PersistencePromise { // Check if watcher isn't up to date or prior results are already held. - return ( - !this.isRemoteUpToVersion(version) || this.heldBatchResults.length > 0 + if (this.heldBatchResults.length > 0) { + return PersistencePromise.resolve(true); + } + + return this.isRemoteUpToVersion(txn, batchVersion).next( + remoteSynced => !remoteSynced ); } @@ -820,9 +956,22 @@ export class LocalStore { ): PersistencePromise { let promiseChain = PersistencePromise.resolve(); for (const batchResult of batchResults) { - promiseChain = promiseChain.next(() => - this.applyWriteToRemoteDocuments(txn, batchResult, documentBuffer) - ); + promiseChain = promiseChain + .next(() => + this.applyWriteToRemoteDocuments(txn, batchResult, documentBuffer) + ) + .next(() => { + // HACK: This should happen in SyncEngine and outside of the + // transaction boundary, but we currently don't expose the Batch ID + // for released batches outside of LocalStore. Due to the way that + // we lock the IndexedDb store, secondary clients will however NOT be + // able to act upon these notifications until after this transaction + // is committed. b/33446471 will remove this reliance. + this.sharedClientState.trackMutationResult( + batchResult.batch.batchId, + 'acknowledged' + ); + }); } return promiseChain.next(() => { return this.removeMutationBatches( @@ -896,4 +1045,28 @@ export class LocalStore { }); return promiseChain; } + + // PORTING NOTE: Multi-tab only. + getQueryForTarget(targetId: TargetId): Promise { + if (this.targetIds[targetId]) { + return Promise.resolve(this.targetIds[targetId].query); + } else { + return this.persistence.runTransaction('Get query data', false, txn => { + return this.queryCache + .getQueryDataForTarget(txn, targetId) + .next(queryData => queryData.query); + }); + } + } + + // PORTING NOTE: Multi-tab only. + getNewDocumentChanges(): Promise { + return this.persistence.runTransaction( + 'Get new document changes', + false, + txn => { + return this.remoteDocuments.getNewDocumentChanges(txn); + } + ); + } } diff --git a/packages/firestore/src/local/memory_mutation_queue.ts b/packages/firestore/src/local/memory_mutation_queue.ts index 85f8a51b112..ba5af955576 100644 --- a/packages/firestore/src/local/memory_mutation_queue.ts +++ b/packages/firestore/src/local/memory_mutation_queue.ts @@ -58,15 +58,6 @@ export class MemoryMutationQueue implements MutationQueue { private batchesByDocumentKey = new SortedSet(DocReference.compareByKey); start(transaction: PersistenceTransaction): PersistencePromise { - // NOTE: The queue may be shutdown / started multiple times, since we - // maintain the queue for the duration of the app session in case a user - // logs out / back in. To behave like the LevelDB-backed MutationQueue (and - // accommodate tests that expect as much), we reset nextBatchId and - // highestAcknowledgedBatchId if the queue is empty. - if (this.mutationQueue.length === 0) { - this.nextBatchId = 1; - this.highestAcknowledgedBatchId = BATCHID_UNKNOWN; - } assert( this.highestAcknowledgedBatchId < this.nextBatchId, 'highestAcknowledgedBatchId must be less than the nextBatchId' @@ -78,12 +69,6 @@ export class MemoryMutationQueue implements MutationQueue { return PersistencePromise.resolve(this.mutationQueue.length === 0); } - getNextBatchId( - transaction: PersistenceTransaction - ): PersistencePromise { - return PersistencePromise.resolve(this.nextBatchId); - } - getHighestAcknowledgedBatchId( transaction: PersistenceTransaction ): PersistencePromise { @@ -174,6 +159,17 @@ export class MemoryMutationQueue implements MutationQueue { return PersistencePromise.resolve(this.findMutationBatch(batchId)); } + lookupMutationKeys( + transaction: PersistenceTransaction, + batchId: BatchId + ): PersistencePromise { + const mutationBatch = this.findMutationBatch(batchId); + assert(mutationBatch != null, 'Failed to find local mutation batch.'); + return PersistencePromise.resolve( + !mutationBatch.isTombstone() ? mutationBatch.keys() : null + ); + } + getNextMutationBatchAfterBatchId( transaction: PersistenceTransaction, batchId: BatchId @@ -404,6 +400,10 @@ export class MemoryMutationQueue implements MutationQueue { return PersistencePromise.resolve(); } + removeCachedMutationKeys(batchId: BatchId): void { + // No-op since the memory mutation queue does not maintain a separate cache. + } + setGarbageCollector(garbageCollector: GarbageCollector | null): void { this.garbageCollector = garbageCollector; } diff --git a/packages/firestore/src/local/memory_persistence.ts b/packages/firestore/src/local/memory_persistence.ts index 19e106d0b68..f5d69e54b4c 100644 --- a/packages/firestore/src/local/memory_persistence.ts +++ b/packages/firestore/src/local/memory_persistence.ts @@ -22,10 +22,15 @@ import { MemoryMutationQueue } from './memory_mutation_queue'; import { MemoryQueryCache } from './memory_query_cache'; import { MemoryRemoteDocumentCache } from './memory_remote_document_cache'; import { MutationQueue } from './mutation_queue'; -import { Persistence, PersistenceTransaction } from './persistence'; +import { + Persistence, + PersistenceTransaction, + PrimaryStateListener +} from './persistence'; import { PersistencePromise } from './persistence_promise'; import { QueryCache } from './query_cache'; import { RemoteDocumentCache } from './remote_document_cache'; +import { ClientId } from './shared_client_state'; const LOG_TAG = 'MemoryPersistence'; @@ -47,6 +52,8 @@ export class MemoryPersistence implements Persistence { private _started = false; + constructor(private readonly clientId: ClientId) {} + async start(): Promise { // No durable state to read on startup. assert(!this._started, 'MemoryPersistence double-started!'); @@ -55,7 +62,6 @@ export class MemoryPersistence implements Persistence { async shutdown(deleteData?: boolean): Promise { // No durable state to ensure is closed on shutdown. - assert(this._started, 'MemoryPersistence shutdown without start!'); this._started = false; } @@ -63,6 +69,21 @@ export class MemoryPersistence implements Persistence { return this._started; } + async getActiveClients(): Promise { + return [this.clientId]; + } + + setPrimaryStateListener( + primaryStateListener: PrimaryStateListener + ): Promise { + // All clients using memory persistence act as primary. + return primaryStateListener(true); + } + + setNetworkEnabled(networkEnabled: boolean): void { + // No op. + } + getMutationQueue(user: User): MutationQueue { let queue = this.mutationQueues[user.toKey()]; if (!queue) { @@ -82,10 +103,13 @@ export class MemoryPersistence implements Persistence { runTransaction( action: string, - operation: (transaction: PersistenceTransaction) => PersistencePromise + requirePrimaryLease: boolean, + transactionOperation: ( + transaction: PersistenceTransaction + ) => PersistencePromise ): Promise { debug(LOG_TAG, 'Starting transaction:', action); - return operation(new MemoryTransaction()).toPromise(); + return transactionOperation(new MemoryTransaction()).toPromise(); } } diff --git a/packages/firestore/src/local/memory_query_cache.ts b/packages/firestore/src/local/memory_query_cache.ts index 6b0166b1d52..1e4a47b24c3 100644 --- a/packages/firestore/src/local/memory_query_cache.ts +++ b/packages/firestore/src/local/memory_query_cache.ts @@ -27,7 +27,8 @@ import { PersistencePromise } from './persistence_promise'; import { QueryCache } from './query_cache'; import { QueryData } from './query_data'; import { ReferenceSet } from './reference_set'; -import { assert } from '../util/assert'; +import { assert, fail } from '../util/assert'; +import { TargetIdGenerator } from '../core/target_id_generator'; export class MemoryQueryCache implements QueryCache { /** @@ -47,24 +48,35 @@ export class MemoryQueryCache implements QueryCache { private targetCount = 0; + private targetIdGenerator = TargetIdGenerator.forQueryCache(); + start(transaction: PersistenceTransaction): PersistencePromise { // Nothing to do. return PersistencePromise.resolve(); } - getLastRemoteSnapshotVersion(): SnapshotVersion { - return this.lastRemoteSnapshotVersion; + getLastRemoteSnapshotVersion( + transaction: PersistenceTransaction + ): PersistencePromise { + return PersistencePromise.resolve(this.lastRemoteSnapshotVersion); } - getHighestTargetId(): TargetId { - return this.highestTargetId; + allocateTargetId( + transaction: PersistenceTransaction + ): PersistencePromise { + const nextTargetId = this.targetIdGenerator.after(this.highestTargetId); + this.highestTargetId = nextTargetId; + return PersistencePromise.resolve(nextTargetId); } - setLastRemoteSnapshotVersion( + setTargetsMetadata( transaction: PersistenceTransaction, - snapshotVersion: SnapshotVersion + highestListenSequenceNumber: number, + lastRemoteSnapshotVersion?: SnapshotVersion ): PersistencePromise { - this.lastRemoteSnapshotVersion = snapshotVersion; + if (lastRemoteSnapshotVersion) { + this.lastRemoteSnapshotVersion = lastRemoteSnapshotVersion; + } return PersistencePromise.resolve(); } @@ -114,8 +126,10 @@ export class MemoryQueryCache implements QueryCache { return PersistencePromise.resolve(); } - get count(): number { - return this.targetCount; + getQueryCount( + transaction: PersistenceTransaction + ): PersistencePromise { + return PersistencePromise.resolve(this.targetCount); } getQueryData( @@ -126,6 +140,15 @@ export class MemoryQueryCache implements QueryCache { return PersistencePromise.resolve(queryData); } + getQueryDataForTarget( + transaction: PersistenceTransaction, + targetId: TargetId + ): never { + // This method is only needed for multi-tab and we can't implement it + // efficiently without additional data structures. + return fail('Not yet implemented.'); + } + addMatchingKeys( txn: PersistenceTransaction, keys: DocumentKeySet, diff --git a/packages/firestore/src/local/memory_remote_document_cache.ts b/packages/firestore/src/local/memory_remote_document_cache.ts index 881ab8c186d..7fc70f4d70d 100644 --- a/packages/firestore/src/local/memory_remote_document_cache.ts +++ b/packages/firestore/src/local/memory_remote_document_cache.ts @@ -16,25 +16,39 @@ import { Query } from '../core/query'; import { + documentKeySet, DocumentMap, documentMap, + MaybeDocumentMap, maybeDocumentMap } from '../model/collections'; -import { Document, MaybeDocument } from '../model/document'; +import { Document, MaybeDocument, NoDocument } from '../model/document'; import { DocumentKey } from '../model/document_key'; import { PersistenceTransaction } from './persistence'; import { PersistencePromise } from './persistence_promise'; import { RemoteDocumentCache } from './remote_document_cache'; +import { SnapshotVersion } from '../core/snapshot_version'; export class MemoryRemoteDocumentCache implements RemoteDocumentCache { private docs = maybeDocumentMap(); + private newDocumentChanges = documentKeySet(); - addEntry( + start(transaction: PersistenceTransaction): PersistencePromise { + // Technically a no-op but we clear the set of changed documents to mimic + // the behavior of the IndexedDb counterpart. + this.newDocumentChanges = documentKeySet(); + return PersistencePromise.resolve(); + } + + addEntries( transaction: PersistenceTransaction, - maybeDocument: MaybeDocument + maybeDocuments: MaybeDocument[] ): PersistencePromise { - this.docs = this.docs.insert(maybeDocument.key, maybeDocument); + for (const maybeDocument of maybeDocuments) { + this.docs = this.docs.insert(maybeDocument.key, maybeDocument); + this.newDocumentChanges = this.newDocumentChanges.add(maybeDocument.key); + } return PersistencePromise.resolve(); } @@ -74,4 +88,22 @@ export class MemoryRemoteDocumentCache implements RemoteDocumentCache { } return PersistencePromise.resolve(results); } + + getNewDocumentChanges( + transaction: PersistenceTransaction + ): PersistencePromise { + let changedDocs = maybeDocumentMap(); + + this.newDocumentChanges.forEach(key => { + changedDocs = changedDocs.insert( + key, + this.docs.get(key) || + new NoDocument(key, SnapshotVersion.forDeletedDoc()) + ); + }); + + this.newDocumentChanges = documentKeySet(); + + return PersistencePromise.resolve(changedDocs); + } } diff --git a/packages/firestore/src/local/mutation_queue.ts b/packages/firestore/src/local/mutation_queue.ts index bc194e984f5..3f829ea42e3 100644 --- a/packages/firestore/src/local/mutation_queue.ts +++ b/packages/firestore/src/local/mutation_queue.ts @@ -42,17 +42,6 @@ export interface MutationQueue extends GarbageSource { /** Returns true if this queue contains no mutation batches. */ checkEmpty(transaction: PersistenceTransaction): PersistencePromise; - /** - * Returns the next BatchId that will be assigned to a new mutation batch. - * - * Callers generally don't care about this value except to test that the - * mutation queue is properly maintaining the invariant that - * highestAcknowledgedBatchId is less than nextBatchId. - */ - getNextBatchId( - transaction: PersistenceTransaction - ): PersistencePromise; - /** * Returns the highest batchId that has been acknowledged. If no batches have * been acknowledged or if there are no batches in the queue this can return @@ -82,19 +71,40 @@ export interface MutationQueue extends GarbageSource { streamToken: ProtoByteString ): PersistencePromise; - /** Creates a new mutation batch and adds it to this mutation queue. */ + /** + * Creates a new mutation batch and adds it to this mutation queue. + * + * TODO(multitab): Make this operation safe to use from secondary clients. + */ addMutationBatch( transaction: PersistenceTransaction, localWriteTime: Timestamp, mutations: Mutation[] ): PersistencePromise; - /** Loads the mutation batch with the given batchId. */ + /** + * Loads the mutation batch with the given batchId. + * + * Multi-Tab Note: This operation is safe to use from secondary clients. + */ lookupMutationBatch( transaction: PersistenceTransaction, batchId: BatchId ): PersistencePromise; + /** + * Returns the document keys for the mutation batch with the given batchId. + * For primary clients, this method returns `null` after + * `removeMutationBatches()` has been called. Secondary clients return a + * cached result until `removeCachedMutationKeys()` is invoked. + * + * Multi-Tab Note: This operation is safe to use from secondary clients. + */ + lookupMutationKeys( + transaction: PersistenceTransaction, + batchId: BatchId + ): PersistencePromise; + /** * Gets the first unacknowledged mutation batch after the passed in batchId * in the mutation queue or null if empty. @@ -208,6 +218,17 @@ export interface MutationQueue extends GarbageSource { batches: MutationBatch[] ): PersistencePromise; + /** + * Clears the cached keys for a mutation batch. This method should be + * called by secondary clients after they process mutation updates. + * + * Note that this method does not have to be called from primary clients as + * the corresponding cache entries are cleared when an acknowledged or + * rejected batch is removed from the mutation queue. + */ + // PORTING NOTE: Multi-tab only + removeCachedMutationKeys(batchId: BatchId): void; + /** * Performs a consistency check, examining the mutation queue for any * leaks, if possible. diff --git a/packages/firestore/src/local/persistence.ts b/packages/firestore/src/local/persistence.ts index d3902ae2f2d..0f77a0ecdbe 100644 --- a/packages/firestore/src/local/persistence.ts +++ b/packages/firestore/src/local/persistence.ts @@ -20,6 +20,7 @@ import { MutationQueue } from './mutation_queue'; import { PersistencePromise } from './persistence_promise'; import { QueryCache } from './query_cache'; import { RemoteDocumentCache } from './remote_document_cache'; +import { ClientId } from './shared_client_state'; /** * Opaque interface representing a persistence transaction. @@ -30,6 +31,18 @@ import { RemoteDocumentCache } from './remote_document_cache'; */ export abstract class PersistenceTransaction {} +/** + * Callback type for primary state notifications. This callback can be + * registered with the persistence layer to get notified when we transition from + * primary to secondary state and vice versa. + * + * Note: Instances can only toggle between Primary and Secondary state if + * IndexedDB persistence is enabled and multiple clients are active. If this + * listener is registered with MemoryPersistence, the callback will be called + * exactly once marking the current instance as Primary. + */ +export type PrimaryStateListener = (isPrimary: boolean) => Promise; + /** * Persistence is the lowest-level shared interface to persistent storage in * Firestore. @@ -66,6 +79,9 @@ export abstract class PersistenceTransaction {} * writes in order to avoid relying on being able to read back uncommitted * writes. */ +// TODO(multitab): Instead of marking methods as multi-tab safe, we should +// point out (and maybe enforce) when methods cannot safely be used from +// secondary tabs. export interface Persistence { /** * Whether or not this persistence instance has been started. @@ -87,6 +103,34 @@ export interface Persistence { */ shutdown(deleteData?: boolean): Promise; + /** + * Registers a listener that gets called when the primary state of the + * instance changes. Upon registering, this listener is invoked immediately + * with the current primary state. + * + * PORTING NOTE: This is only used for Web multi-tab. + */ + setPrimaryStateListener( + primaryStateListener: PrimaryStateListener + ): Promise; + + /** + * Adjusts the current network state in the client's metadata, potentially + * affecting the primary lease. + * + * PORTING NOTE: This is only used for Web multi-tab. + */ + setNetworkEnabled(networkEnabled: boolean): void; + + /** + * Returns the IDs of the clients that are currently active. If multi-tab + * is not supported, returns an array that only contains the local client's + * ID. + * + * PORTING NOTE: This is only used for Web multi-tab. + */ + getActiveClients(): Promise; + /** * Returns a MutationQueue representing the persisted mutations for the * given user. @@ -130,11 +174,16 @@ export interface Persistence { * * @param action A description of the action performed by this transaction, * used for logging. + * @param requirePrimaryLease Whether this transaction can only be executed + * by the primary client. If the primary lease cannot be acquired, the + * transactionOperation will not be run, and the returned promise will be + * rejected with a FAILED_PRECONDITION error. * @param transactionOperation The operation to run inside a transaction. * @return A promise that is resolved once the transaction completes. */ runTransaction( action: string, + requirePrimaryLease: boolean, transactionOperation: ( transaction: PersistenceTransaction ) => PersistencePromise diff --git a/packages/firestore/src/local/query_cache.ts b/packages/firestore/src/local/query_cache.ts index 47b3c410242..af3cfd17c26 100644 --- a/packages/firestore/src/local/query_cache.ts +++ b/packages/firestore/src/local/query_cache.ts @@ -35,13 +35,6 @@ export interface QueryCache extends GarbageSource { */ start(transaction: PersistenceTransaction): PersistencePromise; - /** - * Returns the highest target ID of any query in the cache. Typically called - * during startup to seed a target ID generator and avoid collisions with - * existing queries. If there are no queries in the cache, returns zero. - */ - getHighestTargetId(): TargetId; - /** * A global snapshot version representing the last consistent snapshot we * received from the backend. This is monotonically increasing and any @@ -53,17 +46,22 @@ export interface QueryCache extends GarbageSource { * This is updated whenever our we get a TargetChange with a read_time and * empty target_ids. */ - getLastRemoteSnapshotVersion(): SnapshotVersion; + getLastRemoteSnapshotVersion( + transaction: PersistenceTransaction + ): PersistencePromise; /** - * Set the snapshot version representing the last consistent snapshot received - * from the backend. (see getLastRemoteSnapshotVersion() for more details). + * Set the highest listen sequence number and optionally updates the + * snapshot version of the last consistent snapshot received from the backend + * (see getLastRemoteSnapshotVersion() for more details). * - * @param snapshotVersion The new snapshot version. + * @param highestListenSequenceNumber The new maximum listen sequence number. + * @param lastRemoteSnapshotVersion The new snapshot version. Optional. */ - setLastRemoteSnapshotVersion( + setTargetsMetadata( transaction: PersistenceTransaction, - snapshotVersion: SnapshotVersion + highestListenSequenceNumber: number, + lastRemoteSnapshotVersion?: SnapshotVersion ): PersistencePromise; /** @@ -103,10 +101,15 @@ export interface QueryCache extends GarbageSource { /** * The number of targets currently in the cache. */ - readonly count: number; + // Visible for testing. + getQueryCount( + transaction: PersistenceTransaction + ): PersistencePromise; /** - * Looks up a QueryData entry in the cache. + * Looks up a QueryData entry by query. + * + * Multi-Tab Note: This operation is safe to use from secondary clients. * * @param query The query corresponding to the entry to look up. * @return The cached QueryData entry, or null if the cache has no entry for @@ -117,6 +120,21 @@ export interface QueryCache extends GarbageSource { query: Query ): PersistencePromise; + /** + * Looks up a QueryData entry by target ID. + * + * Multi-Tab Note: This operation is safe to use from secondary clients. + * + * @param targetId The target ID of the QueryData entry to look up. + * @return The cached QueryData entry, or null if the cache has no entry for + * the query. + */ + // PORTING NOTE: Multi-tab only. + getQueryDataForTarget( + txn: PersistenceTransaction, + targetId: TargetId + ): PersistencePromise; + /** * Adds the given document keys to cached query results of the given target * ID. @@ -145,8 +163,23 @@ export interface QueryCache extends GarbageSource { targetId: TargetId ): PersistencePromise; + /** + * Returns the document keys that match the provided target ID. + * + * Multi-Tab Note: This operation is safe to use from secondary clients. + */ getMatchingKeysForTargetId( transaction: PersistenceTransaction, targetId: TargetId ): PersistencePromise; + + /** + * Returns a new target ID that is higher than any query in the cache. If + * there are no queries in the cache, returns the first valid target ID. + * Allocated target IDs are persisted and `allocateTargetId()` will never + * return the same ID twice. + */ + allocateTargetId( + transaction: PersistenceTransaction + ): PersistencePromise; } diff --git a/packages/firestore/src/local/remote_document_cache.ts b/packages/firestore/src/local/remote_document_cache.ts index ceb58d3bf91..2a5f84100cb 100644 --- a/packages/firestore/src/local/remote_document_cache.ts +++ b/packages/firestore/src/local/remote_document_cache.ts @@ -15,7 +15,7 @@ */ import { Query } from '../core/query'; -import { DocumentMap } from '../model/collections'; +import { DocumentMap, MaybeDocumentMap } from '../model/collections'; import { MaybeDocument } from '../model/document'; import { DocumentKey } from '../model/document_key'; @@ -32,16 +32,27 @@ import { PersistencePromise } from './persistence_promise'; */ export interface RemoteDocumentCache { /** - * Adds or replaces an entry in the cache. + * Starts up the remote document cache. + * + * Reads the ID of the last document change from the documentChanges store. + * Existing changes will not be returned as part of + * `getNewDocumentChanges()`. + */ + // PORTING NOTE: This is only used for multi-tab synchronization. + start(transaction: PersistenceTransaction): PersistencePromise; + + /** + * Adds or replaces document entries in the cache. * * The cache key is extracted from `maybeDocument.key`. If there is already a * cache entry for the key, it will be replaced. * - * @param maybeDocument A Document or NoDocument to put in the cache. + * @param maybeDocuments A set of Documents or NoDocuments to put in the + * cache. */ - addEntry( + addEntries( transaction: PersistenceTransaction, - maybeDocument: MaybeDocument + maybeDocuments: MaybeDocument[] ): PersistencePromise; /** Removes the cached entry for the given key (no-op if no entry exists). */ @@ -53,6 +64,8 @@ export interface RemoteDocumentCache { /** * Looks up an entry in the cache. * + * Multi-Tab Note: This operation is safe to use from secondary clients. + * * @param documentKey The key of the entry to look up. * @return The cached Document or NoDocument entry, or null if we have nothing * cached. @@ -70,6 +83,8 @@ export interface RemoteDocumentCache { * * Cached NoDocument entries have no bearing on query results. * + * Multi-Tab Note: This operation is safe to use from secondary clients. + * * @param query The query to match documents against. * @return The set of matching documents. */ @@ -77,4 +92,14 @@ export interface RemoteDocumentCache { transaction: PersistenceTransaction, query: Query ): PersistencePromise; + + /** + * Returns the set of documents that have been updated since the last call. + * If this is the first call, returns the set of changes since client + * initialization. + */ + // PORTING NOTE: This is only used for multi-tab synchronization. + getNewDocumentChanges( + transaction: PersistenceTransaction + ): PersistencePromise; } diff --git a/packages/firestore/src/local/remote_document_change_buffer.ts b/packages/firestore/src/local/remote_document_change_buffer.ts index 184e5301f5f..03129c649a6 100644 --- a/packages/firestore/src/local/remote_document_change_buffer.ts +++ b/packages/firestore/src/local/remote_document_change_buffer.ts @@ -78,17 +78,17 @@ export class RemoteDocumentChangeBuffer { * the provided transaction. */ apply(transaction: PersistenceTransaction): PersistencePromise { - const changes = this.assertChanges(); + const docs: MaybeDocument[] = []; - const promises: Array> = []; + const changes = this.assertChanges(); changes.forEach((key, maybeDoc) => { - promises.push(this.remoteDocumentCache.addEntry(transaction, maybeDoc)); + docs.push(maybeDoc); }); - // We should not be used to buffer any more changes. + // We should not buffer any more changes. this.changes = null; - return PersistencePromise.waitFor(promises); + return this.remoteDocumentCache.addEntries(transaction, docs); } /** Helper to assert this.changes is not null and return it. */ diff --git a/packages/firestore/src/local/shared_client_state.ts b/packages/firestore/src/local/shared_client_state.ts new file mode 100644 index 00000000000..cecff30c576 --- /dev/null +++ b/packages/firestore/src/local/shared_client_state.ts @@ -0,0 +1,1225 @@ +/** + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Code, FirestoreError } from '../util/error'; +import { + BatchId, + MutationBatchState, + OnlineState, + TargetId +} from '../core/types'; +import { assert } from '../util/assert'; +import { debug, error } from '../util/log'; +import { min } from '../util/misc'; +import { SortedSet } from '../util/sorted_set'; +import { isSafeInteger } from '../util/types'; +import * as objUtils from '../util/obj'; +import { User } from '../auth/user'; +import { + QueryTargetState, + SharedClientStateSyncer +} from './shared_client_state_syncer'; +import { AsyncQueue } from '../util/async_queue'; +import { Platform } from '../platform/platform'; +import { batchIdSet, TargetIdSet, targetIdSet } from '../model/collections'; + +const LOG_TAG = 'SharedClientState'; + +// The format of the LocalStorage key that stores the client state is: +// firestore_clients__ +const CLIENT_STATE_KEY_PREFIX = 'firestore_clients'; + +// The format of the LocalStorage key that stores the mutation state is: +// firestore_mutations__ +// (for unauthenticated users) +// or: firestore_mutations___ +// +// 'user_uid' is last to avoid needing to escape '_' characters that it might +// contain. +const MUTATION_BATCH_KEY_PREFIX = 'firestore_mutations'; + +// The format of the LocalStorage key that stores a query target's metadata is: +// firestore_targets__ +const QUERY_TARGET_KEY_PREFIX = 'firestore_targets'; + +// The LocalStorage key that stores the primary tab's online state. +const ONLINE_STATE_KEY = 'firestore_online_state'; + +/** + * A randomly-generated key assigned to each Firestore instance at startup. + */ +export type ClientId = string; + +/** + * A `SharedClientState` keeps track of the global state of the mutations + * and query targets for all active clients with the same persistence key (i.e. + * project ID and FirebaseApp name). It relays local changes to other clients + * and updates its local state as new state is observed. + * + * `SharedClientState` is primarily used for synchronization in Multi-Tab + * environments. Each tab is responsible for registering its active query + * targets and mutations. `SharedClientState` will then notify the listener + * assigned to `.syncEngine` for updates to mutations and queries that + * originated in other clients. + * + * To receive notifications, `.syncEngine` and `.onlineStateHandler` has to be + * assigned before calling `start()`. + */ +export interface SharedClientState { + syncEngine: SharedClientStateSyncer | null; + onlineStateHandler: (onlineState: OnlineState) => void; + + /** Associates a new Mutation Batch ID with the local Firestore client. */ + addLocalPendingMutation(batchId: BatchId): void; + + /** + * Removes a Mutation Batch ID from the local Firestore client. + * + * This method can be called with Batch IDs that are not associated with this + * client, in which case no change takes place. + */ + removeLocalPendingMutation(batchId: BatchId): void; + + /** + * Verifies whether a Mutation Batch ID is associated with the local client. + */ + // Visible for testing. + hasLocalPendingMutation(batchId: BatchId): boolean; + + /** + * Records that a pending mutation has been acknowledged or rejected. + * Called by the primary client to notify secondary clients of mutation + * results as they come back from the backend. + */ + trackMutationResult( + batchId: BatchId, + state: 'acknowledged' | 'rejected', + error?: FirestoreError + ): void; + + /** + * Gets the minimum mutation batch for all active clients. + * + * The implementation for this may require O(n) runtime, where 'n' is the + * number of clients. + */ + getMinimumGlobalPendingMutation(): BatchId | null; + + /** + * Associates a new Query Target ID with the local Firestore client. Returns + * the new query state for the query (which can be 'current' if the query is + * already associated with another tab). + */ + addLocalQueryTarget(targetId: TargetId): QueryTargetState; + + /** Removes a Query Target ID for the local Firestore clients. */ + removeLocalQueryTarget(targetId: TargetId): void; + + /** + * Processes an update to a query target. + * + * Called by the primary client to notify secondary clients of document + * changes or state transitions that affect the provided query target. + */ + trackQueryUpdate( + targetId: TargetId, + state: QueryTargetState, + error?: FirestoreError + ): void; + + /** + * Gets the active Query Targets IDs for all active clients. + * + * The implementation for this may require O(n) runtime, where 'n' is the size + * of the result set. + */ + // Visible for testing + getAllActiveQueryTargets(): SortedSet; + + /** + * Checks whether the provided target ID is currently being listened to by + * any of the active clients. + * + * The implementation may require O(n*log m) runtime, where 'n' is the number + * of clients and 'm' the number of targets. + */ + isActiveQueryTarget(targetId: TargetId): boolean; + + /** + * Starts the SharedClientState, reads existing client data and registers + * listeners for updates to new and existing clients. + */ + start(): Promise; + + /** Shuts down the `SharedClientState` and its listeners. */ + shutdown(): void; + + /** + * Changes the active user and removes all existing user-specific data. The + * user change does not call back into SyncEngine (for example, no mutations + * will be marked as removed). + */ + handleUserChange( + user: User, + removedBatchIds: BatchId[], + addedBatchIds: BatchId[] + ): void; + + /** Changes the shared online state of all clients. */ + setOnlineState(onlineState: OnlineState): void; +} + +/** + * The JSON representation of a mutation batch's metadata as used during + * LocalStorage serialization. The UserId and BatchId is omitted as it is + * encoded as part of the key. + */ +interface MutationMetadataSchema { + state: MutationBatchState; + error?: { code: string; message: string }; // Only set when state === 'rejected' +} + +/** + * Holds the state of a mutation batch, including its user ID, batch ID and + * whether the batch is 'pending', 'acknowledged' or 'rejected'. + */ +// Visible for testing +export class MutationMetadata { + constructor( + readonly user: User, + readonly batchId: BatchId, + readonly state: MutationBatchState, + readonly error?: FirestoreError + ) { + assert( + (error !== undefined) === (state === 'rejected'), + `MutationMetadata must contain an error iff state is 'rejected'` + ); + } + + /** + * Parses a MutationMetadata from its JSON representation in LocalStorage. + * Logs a warning and returns null if the format of the data is not valid. + */ + static fromLocalStorageEntry( + user: User, + batchId: BatchId, + value: string + ): MutationMetadata | null { + const mutationBatch = JSON.parse(value) as MutationMetadataSchema; + + let validData = + typeof mutationBatch === 'object' && + ['pending', 'acknowledged', 'rejected'].indexOf(mutationBatch.state) !== + -1 && + (mutationBatch.error === undefined || + typeof mutationBatch.error === 'object'); + + let firestoreError = undefined; + + if (validData && mutationBatch.error) { + validData = + typeof mutationBatch.error.message === 'string' && + typeof mutationBatch.error.code === 'string'; + if (validData) { + firestoreError = new FirestoreError( + mutationBatch.error.code as Code, + mutationBatch.error.message + ); + } + } + + if (validData) { + return new MutationMetadata( + user, + batchId, + mutationBatch.state, + firestoreError + ); + } else { + error( + LOG_TAG, + `Failed to parse mutation state for ID '${batchId}': ${value}` + ); + return null; + } + } + + toLocalStorageJSON(): string { + const batchMetadata: MutationMetadataSchema = { + state: this.state + }; + + if (this.error) { + batchMetadata.error = { + code: this.error.code, + message: this.error.message + }; + } + + return JSON.stringify(batchMetadata); + } +} + +/** + * The JSON representation of a query target's state as used during LocalStorage + * serialization. The TargetId is omitted as it is encoded as part of the key. + */ +interface QueryTargetStateSchema { + lastUpdateTime: number; + state: QueryTargetState; + error?: { code: string; message: string }; // Only set when state === 'rejected' +} + +/** + * Holds the state of a query target, including its target ID and whether the + * target is 'not-current', 'current' or 'rejected'. + */ +// Visible for testing +export class QueryTargetMetadata { + constructor( + readonly targetId: TargetId, + readonly lastUpdateTime: Date, + readonly state: QueryTargetState, + readonly error?: FirestoreError + ) { + assert( + (error !== undefined) === (state === 'rejected'), + `QueryTargetMetadata must contain an error iff state is 'rejected'` + ); + } + + /** + * Parses a QueryTargetMetadata from its JSON representation in LocalStorage. + * Logs a warning and returns null if the format of the data is not valid. + */ + static fromLocalStorageEntry( + targetId: TargetId, + value: string + ): QueryTargetMetadata | null { + const targetState = JSON.parse(value) as QueryTargetStateSchema; + + let validData = + typeof targetState === 'object' && + isSafeInteger(targetState.lastUpdateTime) && + ['not-current', 'current', 'rejected'].indexOf(targetState.state) !== + -1 && + (targetState.error === undefined || + typeof targetState.error === 'object'); + + let firestoreError = undefined; + + if (validData && targetState.error) { + validData = + typeof targetState.error.message === 'string' && + typeof targetState.error.code === 'string'; + if (validData) { + firestoreError = new FirestoreError( + targetState.error.code as Code, + targetState.error.message + ); + } + } + + if (validData) { + return new QueryTargetMetadata( + targetId, + new Date(targetState.lastUpdateTime), + targetState.state, + firestoreError + ); + } else { + error( + LOG_TAG, + `Failed to parse target state for ID '${targetId}': ${value}` + ); + return null; + } + } + + toLocalStorageJSON(): string { + const targetState: QueryTargetStateSchema = { + lastUpdateTime: this.lastUpdateTime.getTime(), + state: this.state + }; + + if (this.error) { + targetState.error = { + code: this.error.code, + message: this.error.message + }; + } + + return JSON.stringify(targetState); + } +} + +/** + * The JSON representation of a clients's metadata as used during LocalStorage + * serialization. The ClientId is omitted here as it is encoded as part of the + * key. + */ +interface ClientStateSchema { + lastUpdateTime: number; + activeTargetIds: number[]; + minMutationBatchId: number | null; + maxMutationBatchId: number | null; +} + +/** + * Metadata state of a single client. Includes query targets, the minimum + * pending and maximum pending mutation batch ID, as well as the last update + * time of this state. + */ +// Visible for testing. +export interface ClientState { + readonly activeTargetIds: TargetIdSet; + readonly lastUpdateTime: Date; + readonly maxMutationBatchId: BatchId | null; + readonly minMutationBatchId: BatchId | null; +} + +/** + * This class represents the immutable ClientState for a client read from + * LocalStorage. It contains the list of its active query targets and the range + * of its pending mutation batch IDs. + */ +class RemoteClientState implements ClientState { + private constructor( + readonly clientId: ClientId, + readonly lastUpdateTime: Date, + readonly activeTargetIds: TargetIdSet, + readonly minMutationBatchId: BatchId | null, + readonly maxMutationBatchId: BatchId | null + ) {} + + /** + * Parses a RemoteClientState from the JSON representation in LocalStorage. + * Logs a warning and returns null if the format of the data is not valid. + */ + static fromLocalStorageEntry( + clientId: ClientId, + value: string + ): RemoteClientState | null { + const clientState = JSON.parse(value) as ClientStateSchema; + + let validData = + typeof clientState === 'object' && + isSafeInteger(clientState.lastUpdateTime) && + clientState.activeTargetIds instanceof Array && + (clientState.minMutationBatchId === null || + isSafeInteger(clientState.minMutationBatchId)) && + (clientState.maxMutationBatchId === null || + isSafeInteger(clientState.maxMutationBatchId)); + + let activeTargetIdsSet = targetIdSet(); + + for (let i = 0; validData && i < clientState.activeTargetIds.length; ++i) { + validData = isSafeInteger(clientState.activeTargetIds[i]); + activeTargetIdsSet = activeTargetIdsSet.add( + clientState.activeTargetIds[i] + ); + } + + if (validData) { + return new RemoteClientState( + clientId, + new Date(clientState.lastUpdateTime), + activeTargetIdsSet, + clientState.minMutationBatchId, + clientState.maxMutationBatchId + ); + } else { + error( + LOG_TAG, + `Failed to parse client data for instance '${clientId}': ${value}` + ); + return null; + } + } +} + +/** + * The JSON representation of the system's online state, as written by the + * primary client. + */ +export interface SharedOnlineStateSchema { + /** + * The clientId of the client that wrote this onlineState value. Tracked so + * that on startup, clients can check if this client is still active when + * determining whether to apply this value or not. + */ + readonly clientId: string; + readonly onlineState: string; +} + +/** + * This class represents the online state for all clients participating in + * multi-tab. The online state is only written to by the primary client, and + * used in secondary clients to update their query views. + */ +export class SharedOnlineState { + constructor(readonly clientId: string, readonly onlineState: OnlineState) {} + + /** + * Parses a SharedOnlineState from its JSON representation in LocalStorage. + * Logs a warning and returns null if the format of the data is not valid. + */ + static fromLocalStorageEntry(value: string): SharedOnlineState | null { + const onlineState = JSON.parse(value) as SharedOnlineStateSchema; + + const validData = + typeof onlineState === 'object' && + OnlineState[onlineState.onlineState] !== undefined && + typeof onlineState.clientId === 'string'; + + if (validData) { + return new SharedOnlineState( + onlineState.clientId, + OnlineState[onlineState.onlineState] + ); + } else { + error(LOG_TAG, `Failed to parse online state: ${value}`); + return null; + } + } +} + +/** + * Metadata state of the local client. Unlike `RemoteClientState`, this class is + * mutable and keeps track of all pending mutations, which allows us to + * update the range of pending mutation batch IDs as new mutations are added or + * removed. + * + * The data in `LocalClientState` is not read from LocalStorage and instead + * updated via its instance methods. The updated state can be serialized via + * `toLocalStorageJSON()`. + */ +// Visible for testing. +export class LocalClientState implements ClientState { + activeTargetIds = targetIdSet(); + lastUpdateTime: Date; + + private pendingBatchIds = batchIdSet(); + + constructor() { + this.lastUpdateTime = new Date(); + } + + get minMutationBatchId(): BatchId | null { + return this.pendingBatchIds.first(); + } + + get maxMutationBatchId(): BatchId | null { + return this.pendingBatchIds.last(); + } + + addPendingMutation(batchId: BatchId): void { + assert( + !this.pendingBatchIds.has(batchId), + `Batch with ID '${batchId}' already pending.` + ); + this.pendingBatchIds = this.pendingBatchIds.add(batchId); + } + + removePendingMutation(batchId: BatchId): void { + this.pendingBatchIds = this.pendingBatchIds.delete(batchId); + } + + hasLocalPendingMutation(batchId: BatchId): boolean { + return this.pendingBatchIds.has(batchId); + } + + addQueryTarget(targetId: TargetId): void { + assert( + !this.activeTargetIds.has(targetId), + `Target with ID '${targetId}' already active.` + ); + this.activeTargetIds = this.activeTargetIds.add(targetId); + } + + removeQueryTarget(targetId: TargetId): void { + this.activeTargetIds = this.activeTargetIds.delete(targetId); + } + + /** Sets the update time to the current time. */ + refreshLastUpdateTime(): void { + this.lastUpdateTime = new Date(); + } + + /** + * Converts this entry into a JSON-encoded format we can use for LocalStorage. + * Does not encode `clientId` as it is part of the key in LocalStorage. + */ + toLocalStorageJSON(): string { + const data: ClientStateSchema = { + lastUpdateTime: this.lastUpdateTime.getTime(), + activeTargetIds: this.activeTargetIds.toArray(), + minMutationBatchId: this.minMutationBatchId, + maxMutationBatchId: this.maxMutationBatchId + }; + return JSON.stringify(data); + } +} + +/** + * `WebStorageSharedClientState` uses WebStorage (window.localStorage) as the + * backing store for the SharedClientState. It keeps track of all active + * clients and supports modifications of the local client's data. + */ +// TODO(multitab): Rename all usages of LocalStorage to WebStorage to better differentiate from LocalClient. +export class WebStorageSharedClientState implements SharedClientState { + syncEngine: SharedClientStateSyncer | null = null; + onlineStateHandler: (onlineState: OnlineState) => void | null = null; + + private readonly storage: Storage; + private readonly localClientStorageKey: string; + private readonly activeClients: { [key: string]: ClientState } = {}; + private readonly storageListener = this.handleLocalStorageEvent.bind(this); + private readonly clientStateKeyRe: RegExp; + private readonly mutationBatchKeyRe: RegExp; + private readonly queryTargetKeyRe: RegExp; + private started = false; + private currentUser: User; + + /** + * Captures WebStorage events that occur before `start()` is called. These + * events are replayed once `WebStorageSharedClientState` is started. + */ + private earlyEvents: StorageEvent[] = []; + + constructor( + private readonly queue: AsyncQueue, + private readonly platform: Platform, + private readonly persistenceKey: string, + private readonly localClientId: ClientId, + initialUser: User + ) { + if (!WebStorageSharedClientState.isAvailable(this.platform)) { + throw new FirestoreError( + Code.UNIMPLEMENTED, + 'LocalStorage is not available on this platform.' + ); + } + this.storage = this.platform.window.localStorage; + this.currentUser = initialUser; + this.localClientStorageKey = this.toLocalStorageClientStateKey( + this.localClientId + ); + this.activeClients[this.localClientId] = new LocalClientState(); + + // Escape the special characters mentioned here: + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions + const escapedPersistenceKey = persistenceKey.replace( + /[.*+?^${}()|[\]\\]/g, + '\\$&' + ); + + this.clientStateKeyRe = new RegExp( + `^${CLIENT_STATE_KEY_PREFIX}_${escapedPersistenceKey}_([^_]*)$` + ); + this.mutationBatchKeyRe = new RegExp( + `^${MUTATION_BATCH_KEY_PREFIX}_${escapedPersistenceKey}_(\\d+)(?:_(.*))?$` + ); + this.queryTargetKeyRe = new RegExp( + `^${QUERY_TARGET_KEY_PREFIX}_${escapedPersistenceKey}_(\\d+)$` + ); + + // Rather than adding the storage observer during start(), we add the + // storage observer during initialization. This ensures that we collect + // events before other components populate their initial state (during their + // respective start() calls). Otherwise, we might for example miss a + // mutation that is added after LocalStore's start() processed the existing + // mutations but before we observe WebStorage events. + this.platform.window.addEventListener('storage', this.storageListener); + } + + /** Returns 'true' if LocalStorage is available in the current environment. */ + static isAvailable(platform: Platform): boolean { + return platform.window && platform.window.localStorage != null; + } + + // TOOD(multitab): Register the mutations that are already pending at client + // startup. + async start(): Promise { + assert(!this.started, 'WebStorageSharedClientState already started'); + assert( + this.syncEngine !== null, + 'syncEngine property must be set before calling start()' + ); + assert( + this.onlineStateHandler !== null, + 'onlineStateHandler property must be set before calling start()' + ); + + // Retrieve the list of existing clients to backfill the data in + // SharedClientState. + const existingClients = await this.syncEngine.getActiveClients(); + + for (const clientId of existingClients) { + if (clientId === this.localClientId) { + continue; + } + + const storageItem = this.getItem( + this.toLocalStorageClientStateKey(clientId) + ); + if (storageItem) { + const clientState = RemoteClientState.fromLocalStorageEntry( + clientId, + storageItem + ); + if (clientState) { + this.activeClients[clientState.clientId] = clientState; + } + } + } + + this.persistClientState(); + + // Check if there is an existing online state and call the callback handler + // if applicable. + const onlineStateJSON = this.storage.getItem(ONLINE_STATE_KEY); + if (onlineStateJSON) { + const onlineState = this.fromLocalStorageOnlineState(onlineStateJSON); + if (onlineState) { + this.handleOnlineStateEvent(onlineState); + } + } + + for (const event of this.earlyEvents) { + this.handleLocalStorageEvent(event); + } + + this.earlyEvents = []; + this.started = true; + } + + // TODO(multitab): Return BATCHID_UNKNOWN instead of null + getMinimumGlobalPendingMutation(): BatchId | null { + let minMutationBatch: number | null = null; + objUtils.forEach(this.activeClients, (key, value) => { + minMutationBatch = min(minMutationBatch, value.minMutationBatchId); + }); + return minMutationBatch; + } + + getAllActiveQueryTargets(): TargetIdSet { + let activeTargets = targetIdSet(); + objUtils.forEach(this.activeClients, (key, value) => { + activeTargets = activeTargets.unionWith(value.activeTargetIds); + }); + return activeTargets; + } + + isActiveQueryTarget(targetId: TargetId): boolean { + // This is not using `obj.forEach` since `forEach` doesn't support early + // return. + for (const clientId in this.activeClients) { + if (this.activeClients.hasOwnProperty(clientId)) { + if (this.activeClients[clientId].activeTargetIds.has(targetId)) { + return true; + } + } + } + return false; + } + + addLocalPendingMutation(batchId: BatchId): void { + this.localClientState.addPendingMutation(batchId); + this.persistMutationState(batchId, 'pending'); + this.persistClientState(); + } + + removeLocalPendingMutation(batchId: BatchId): void { + this.localClientState.removePendingMutation(batchId); + this.persistClientState(); + } + + hasLocalPendingMutation(batchId: BatchId): boolean { + return this.localClientState.hasLocalPendingMutation(batchId); + } + + trackMutationResult( + batchId: BatchId, + state: 'acknowledged' | 'rejected', + error?: FirestoreError + ): void { + // Only persist the mutation state if at least one client is still + // interested in this mutation. The mutation might have already been + // removed by a client that is no longer active. + if (batchId >= this.getMinimumGlobalPendingMutation()) { + this.persistMutationState(batchId, state, error); + } + } + + addLocalQueryTarget(targetId: TargetId): QueryTargetState { + let queryState: QueryTargetState = 'not-current'; + + // Lookup an existing query state if the target ID was already registered + // by another tab + if (this.isActiveQueryTarget(targetId)) { + const storageItem = this.storage.getItem( + this.toLocalStorageQueryTargetMetadataKey(targetId) + ); + + if (storageItem) { + const metadata = QueryTargetMetadata.fromLocalStorageEntry( + targetId, + storageItem + ); + if (metadata) { + queryState = metadata.state; + } + } + } + + this.localClientState.addQueryTarget(targetId); + this.persistClientState(); + + return queryState; + } + + removeLocalQueryTarget(targetId: TargetId): void { + this.localClientState.removeQueryTarget(targetId); + this.persistClientState(); + // TODO(multitab): Remove the query state from Local Storage. + } + + trackQueryUpdate( + targetId: TargetId, + state: QueryTargetState, + error?: FirestoreError + ): void { + this.persistQueryTargetState(targetId, state, error); + } + + handleUserChange( + user: User, + removedBatchIds: BatchId[], + addedBatchIds: BatchId[] + ): void { + removedBatchIds.forEach(batchId => { + this.removeLocalPendingMutation(batchId); + }); + this.currentUser = user; + addedBatchIds.forEach(batchId => { + this.addLocalPendingMutation(batchId); + }); + } + + setOnlineState(onlineState: OnlineState): void { + this.persistOnlineState(onlineState); + } + + shutdown(): void { + assert( + this.started, + 'WebStorageSharedClientState.shutdown() called when not started' + ); + this.platform.window.removeEventListener('storage', this.storageListener); + this.removeItem(this.localClientStorageKey); + this.started = false; + } + + private getItem(key: string): string | null { + const value = this.storage.getItem(key); + debug(LOG_TAG, 'READ', key, value); + return value; + } + + private setItem(key: string, value: string): void { + debug(LOG_TAG, 'SET', key, value); + this.storage.setItem(key, value); + } + + private removeItem(key: string): void { + debug(LOG_TAG, 'REMOVE', key); + this.storage.removeItem(key); + } + + private handleLocalStorageEvent(event: StorageEvent): void { + if (event.storageArea === this.storage) { + // TODO(multitab): This assert will likely become invalid as we add garbage + // collection. + assert( + event.key !== this.localClientStorageKey, + 'Received LocalStorage notification for local change.' + ); + + debug(LOG_TAG, 'EVENT', event.key, event.newValue); + + this.queue.enqueueAndForget(async () => { + if (!this.started) { + this.earlyEvents.push(event); + return; + } + + if (this.clientStateKeyRe.test(event.key)) { + if (event.newValue != null) { + const clientState = this.fromLocalStorageClientState( + event.key, + event.newValue + ); + if (clientState) { + return this.handleClientStateEvent( + clientState.clientId, + clientState + ); + } + } else { + const clientId = this.fromLocalStorageClientStateKey(event.key); + return this.handleClientStateEvent(clientId, null); + } + } else if (this.mutationBatchKeyRe.test(event.key)) { + if (event.newValue !== null) { + const mutationMetadata = this.fromLocalStorageMutationMetadata( + event.key, + event.newValue + ); + if (mutationMetadata) { + return this.handleMutationBatchEvent(mutationMetadata); + } + } + } else if (this.queryTargetKeyRe.test(event.key)) { + if (event.newValue !== null) { + const queryTargetMetadata = this.fromLocalStorageQueryTargetMetadata( + event.key, + event.newValue + ); + if (queryTargetMetadata) { + return this.handleQueryTargetEvent(queryTargetMetadata); + } + } + } else if (event.key === ONLINE_STATE_KEY) { + if (event.newValue !== null) { + const onlineState = this.fromLocalStorageOnlineState( + event.newValue + ); + if (onlineState) { + return this.handleOnlineStateEvent(onlineState); + } + } + } + }); + } + } + + private get localClientState(): LocalClientState { + return this.activeClients[this.localClientId] as LocalClientState; + } + + private persistClientState(): void { + // TODO(multitab): Consider rate limiting/combining state updates for + // clients that frequently update their client state. + this.localClientState.refreshLastUpdateTime(); + this.setItem( + this.localClientStorageKey, + this.localClientState.toLocalStorageJSON() + ); + } + + private persistMutationState( + batchId: BatchId, + state: MutationBatchState, + error?: FirestoreError + ): void { + const mutationState = new MutationMetadata( + this.currentUser, + batchId, + state, + error + ); + + let mutationKey = `${MUTATION_BATCH_KEY_PREFIX}_${ + this.persistenceKey + }_${batchId}`; + + if (this.currentUser.isAuthenticated()) { + mutationKey += `_${this.currentUser.uid}`; + } + + this.setItem(mutationKey, mutationState.toLocalStorageJSON()); + } + + private persistOnlineState(onlineState: OnlineState): void { + const entry: SharedOnlineStateSchema = { + clientId: this.localClientId, + onlineState: OnlineState[onlineState] + }; + this.storage.setItem(ONLINE_STATE_KEY, JSON.stringify(entry)); + } + + private persistQueryTargetState( + targetId: TargetId, + state: QueryTargetState, + error?: FirestoreError + ): void { + const targetMetadata = new QueryTargetMetadata( + targetId, + /* lastUpdateTime= */ new Date(), + state, + error + ); + + const targetKey = `${QUERY_TARGET_KEY_PREFIX}_${ + this.persistenceKey + }_${targetId}`; + + this.setItem(targetKey, targetMetadata.toLocalStorageJSON()); + } + + /** Assembles the key for a client state in LocalStorage */ + private toLocalStorageClientStateKey(clientId: ClientId): string { + assert( + clientId.indexOf('_') === -1, + `Client key cannot contain '_', but was '${clientId}'` + ); + + return `${CLIENT_STATE_KEY_PREFIX}_${this.persistenceKey}_${clientId}`; + } + + /** Assembles the key for a query state in LocalStorage */ + private toLocalStorageQueryTargetMetadataKey(targetId: TargetId): string { + return `${QUERY_TARGET_KEY_PREFIX}_${this.persistenceKey}_${targetId}`; + } + + /** + * Parses a client state key in LocalStorage. Returns null if the key does not + * match the expected key format. + */ + private fromLocalStorageClientStateKey(key: string): ClientId | null { + const match = this.clientStateKeyRe.exec(key); + return match ? match[1] : null; + } + + /** + * Parses a client state in LocalStorage. Returns 'null' if the value could + * not be parsed. + */ + private fromLocalStorageClientState( + key: string, + value: string + ): RemoteClientState | null { + const clientId = this.fromLocalStorageClientStateKey(key); + assert(clientId !== null, `Cannot parse client state key '${key}'`); + return RemoteClientState.fromLocalStorageEntry(clientId, value); + } + + /** + * Parses a mutation batch state in LocalStorage. Returns 'null' if the value + * could not be parsed. + */ + private fromLocalStorageMutationMetadata( + key: string, + value: string + ): MutationMetadata | null { + const match = this.mutationBatchKeyRe.exec(key); + assert(match !== null, `Cannot parse mutation batch key '${key}'`); + + const batchId = Number(match[1]); + const userId = match[2] !== undefined ? match[2] : null; + return MutationMetadata.fromLocalStorageEntry( + new User(userId), + batchId, + value + ); + } + + /** + * Parses a query target state from LocalStorage. Returns 'null' if the value + * could not be parsed. + */ + private fromLocalStorageQueryTargetMetadata( + key: string, + value: string + ): QueryTargetMetadata | null { + const match = this.queryTargetKeyRe.exec(key); + assert(match !== null, `Cannot parse query target key '${key}'`); + + const targetId = Number(match[1]); + return QueryTargetMetadata.fromLocalStorageEntry(targetId, value); + } + + /** + * Parses an online state from LocalStorage. Returns 'null' if the value + * could not be parsed. + */ + private fromLocalStorageOnlineState(value: string): SharedOnlineState | null { + return SharedOnlineState.fromLocalStorageEntry(value); + } + + private async handleMutationBatchEvent( + mutationBatch: MutationMetadata + ): Promise { + if (mutationBatch.user.uid !== this.currentUser.uid) { + debug( + LOG_TAG, + `Ignoring mutation for non-active user ${mutationBatch.user.uid}` + ); + return; + } + + return this.syncEngine.applyBatchState( + mutationBatch.batchId, + mutationBatch.state, + mutationBatch.error + ); + } + + private handleQueryTargetEvent( + targetMetadata: QueryTargetMetadata + ): Promise { + return this.syncEngine.applyTargetState( + targetMetadata.targetId, + targetMetadata.state, + targetMetadata.error + ); + } + + private handleClientStateEvent( + clientId: ClientId, + clientState: RemoteClientState | null + ): Promise { + const existingTargets = this.getAllActiveQueryTargets(); + + if (clientState) { + this.activeClients[clientId] = clientState; + } else { + delete this.activeClients[clientId]; + } + + const newTargets = this.getAllActiveQueryTargets(); + + const addedTargets: TargetId[] = []; + const removedTargets: TargetId[] = []; + + newTargets.forEach(async targetId => { + if (!existingTargets.has(targetId)) { + addedTargets.push(targetId); + } + }); + + existingTargets.forEach(async targetId => { + if (!newTargets.has(targetId)) { + removedTargets.push(targetId); + } + }); + + return this.syncEngine.applyActiveTargetsChange( + addedTargets, + removedTargets + ); + } + + private handleOnlineStateEvent(onlineState: SharedOnlineState): void { + // We check whether the client that wrote this online state is still active + // by comparing its client ID to the list of clients kept active in + // IndexedDb. If a client does not update their IndexedDb client state + // within 5 seconds, it is considered inactive and we don't emit an online + // state event. + if (this.activeClients[onlineState.clientId]) { + this.onlineStateHandler(onlineState.onlineState); + } + } +} + +/** + * `MemorySharedClientState` is a simple implementation of SharedClientState for + * clients using memory persistence. The state in this class remains fully + * isolated and no synchronization is performed. + */ +export class MemorySharedClientState implements SharedClientState { + private localState = new LocalClientState(); + private queryState: { [targetId: number]: QueryTargetState } = {}; + + syncEngine: SharedClientStateSyncer | null = null; + onlineStateHandler: (onlineState: OnlineState) => void | null = null; + + addLocalPendingMutation(batchId: BatchId): void { + this.localState.addPendingMutation(batchId); + } + + removeLocalPendingMutation(batchId: BatchId): void { + this.localState.removePendingMutation(batchId); + } + + hasLocalPendingMutation(batchId: BatchId): boolean { + return this.localState.hasLocalPendingMutation(batchId); + } + + trackMutationResult( + batchId: BatchId, + state: 'acknowledged' | 'rejected', + error?: FirestoreError + ): void { + // No op. + } + + getMinimumGlobalPendingMutation(): BatchId | null { + return this.localState.minMutationBatchId; + } + + addLocalQueryTarget(targetId: TargetId): QueryTargetState { + this.localState.addQueryTarget(targetId); + return this.queryState[targetId] || 'not-current'; + } + + trackQueryUpdate( + targetId: TargetId, + state: QueryTargetState, + error?: FirestoreError + ): void { + this.queryState[targetId] = state; + } + + removeLocalQueryTarget(targetId: TargetId): void { + this.localState.removeQueryTarget(targetId); + delete this.queryState[targetId]; + } + + getAllActiveQueryTargets(): TargetIdSet { + return this.localState.activeTargetIds; + } + + isActiveQueryTarget(targetId: TargetId): boolean { + return this.localState.activeTargetIds.has(targetId); + } + + start(): Promise { + this.localState = new LocalClientState(); + return Promise.resolve(); + } + + handleUserChange( + user: User, + removedBatchIds: BatchId[], + addedBatchIds: BatchId[] + ): void { + removedBatchIds.forEach(batchId => { + this.localState.removePendingMutation(batchId); + }); + addedBatchIds.forEach(batchId => { + this.localState.addPendingMutation(batchId); + }); + } + + setOnlineState(onlineState: OnlineState): void { + // No op. + } + + shutdown(): void {} +} diff --git a/packages/firestore/src/local/shared_client_state_syncer.ts b/packages/firestore/src/local/shared_client_state_syncer.ts new file mode 100644 index 00000000000..4fa4c545699 --- /dev/null +++ b/packages/firestore/src/local/shared_client_state_syncer.ts @@ -0,0 +1,56 @@ +/** + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BatchId, MutationBatchState, TargetId } from '../core/types'; +import { FirestoreError } from '../util/error'; +import { ClientId } from './shared_client_state'; + +/** The different states of a watch target. */ +export type QueryTargetState = 'not-current' | 'current' | 'rejected'; + +/** + * An interface that describes the actions the SharedClientState class needs to + * perform on a cooperating synchronization engine. + */ +export interface SharedClientStateSyncer { + // TODO(multitab): Consider different names for these methods that convey + // that these method are used in multi-tab to load existing batches from + // persistence (a possible name for `applyBatchState` could be + // `applyBatchFromPersistence`). + + /** Applies a mutation state to an existing batch. */ + applyBatchState( + batchId: BatchId, + state: MutationBatchState, + error?: FirestoreError + ): Promise; + + /** Applies a query target change from a different tab. */ + applyTargetState( + targetId: TargetId, + state: QueryTargetState, + error?: FirestoreError + ): Promise; + + /** Adds or removes Watch targets for queries from different tabs. */ + applyActiveTargetsChange( + added: TargetId[], + removed: TargetId[] + ): Promise; + + /** Returns the IDs of the clients that are currently active. */ + getActiveClients(): Promise; +} diff --git a/packages/firestore/src/local/simple_db.ts b/packages/firestore/src/local/simple_db.ts index 34a2d28ea9f..257038df185 100644 --- a/packages/firestore/src/local/simple_db.ts +++ b/packages/firestore/src/local/simple_db.ts @@ -303,7 +303,7 @@ export class SimpleDbTransaction { if (!this.aborted) { debug( LOG_TAG, - 'Aborting transaction: %s', + 'Aborting transaction:', error ? error.message : 'Client-initiated abort' ); this.aborted = true; @@ -369,6 +369,19 @@ export class SimpleDbStore< return wrapRequest(request); } + /** + * Adds a new value into an Object Store and returns the new key. Similar to + * IndexedDb's `add()`, this method will fail on primary key collisions. + * + * @param value The object to write. + * @return The key of the value to add. + */ + add(value: ValueType): PersistencePromise { + debug(LOG_TAG, 'ADD', this.store.name, value, value); + const request = this.store.add(value as ValueType); + return wrapRequest(request); + } + /** * Gets the object with the specified key from the specified store, or null * if no object exists with the specified key. diff --git a/packages/firestore/src/model/collections.ts b/packages/firestore/src/model/collections.ts index e89e7061112..b6bef779b08 100644 --- a/packages/firestore/src/model/collections.ts +++ b/packages/firestore/src/model/collections.ts @@ -20,6 +20,8 @@ import { SortedSet } from '../util/sorted_set'; import { Document, MaybeDocument } from './document'; import { DocumentKey } from './document_key'; +import { primitiveComparator } from '../util/misc'; +import { BatchId, TargetId } from '../core/types'; /** Miscellaneous collection types / constants. */ @@ -52,3 +54,15 @@ const EMPTY_DOCUMENT_KEY_SET = new SortedSet(DocumentKey.comparator); export function documentKeySet(): DocumentKeySet { return EMPTY_DOCUMENT_KEY_SET; } + +export type TargetIdSet = SortedSet; +const EMPTY_TARGET_ID_SET = new SortedSet(primitiveComparator); +export function targetIdSet(): SortedSet { + return EMPTY_TARGET_ID_SET; +} + +export type BatchIdSet = SortedSet; +const EMPTY_BATCH_ID_SET = new SortedSet(primitiveComparator); +export function batchIdSet(): SortedSet { + return EMPTY_BATCH_ID_SET; +} diff --git a/packages/firestore/src/platform/platform.ts b/packages/firestore/src/platform/platform.ts index cb2ff6dc5b7..2dda1d34a54 100644 --- a/packages/firestore/src/platform/platform.ts +++ b/packages/firestore/src/platform/platform.ts @@ -27,6 +27,8 @@ import { AnyJs } from '../util/misc'; * * An implementation of this must be provided at compile time for the platform. */ +// TODO: Consider only exposing the APIs of 'document' and 'window' that we +// use in our client. export interface Platform { loadConnection(databaseInfo: DatabaseInfo): Promise; newSerializer(databaseId: DatabaseId): JsonProtoSerializer; @@ -40,6 +42,12 @@ export interface Platform { /** Converts a binary string to a Base64 encoded string. */ btoa(raw: string): string; + /** The Platform's 'window' implementation or null if not available. */ + readonly window: Window | null; + + /** The Platform's 'document' implementation or null if not available. */ + readonly document: Document | null; + /** True if and only if the Base64 conversion functions are available. */ readonly base64Available: boolean; diff --git a/packages/firestore/src/platform_browser/browser_platform.ts b/packages/firestore/src/platform_browser/browser_platform.ts index 157af2d7ee0..b3ee09a939a 100644 --- a/packages/firestore/src/platform_browser/browser_platform.ts +++ b/packages/firestore/src/platform_browser/browser_platform.ts @@ -27,6 +27,10 @@ export class BrowserPlatform implements Platform { readonly emptyByteString = ''; + readonly document = document; + + readonly window = window; + constructor() { this.base64Available = typeof atob !== 'undefined'; } diff --git a/packages/firestore/src/platform_node/node_platform.ts b/packages/firestore/src/platform_node/node_platform.ts index 8b507ddf6c5..35312ff87ef 100644 --- a/packages/firestore/src/platform_node/node_platform.ts +++ b/packages/firestore/src/platform_node/node_platform.ts @@ -31,6 +31,16 @@ export class NodePlatform implements Platform { readonly emptyByteString = new Uint8Array(0); + readonly document = null; + + get window(): Window | null { + if (process.env.USE_MOCK_PERSISTENCE === 'YES') { + return window; + } + + return null; + } + loadConnection(databaseInfo: DatabaseInfo): Promise { const protos = loadProtos(); return Promise.resolve(new GrpcConnection(protos, databaseInfo)); diff --git a/packages/firestore/src/remote/datastore.ts b/packages/firestore/src/remote/datastore.ts index a958e8076a5..73a2fee80a1 100644 --- a/packages/firestore/src/remote/datastore.ts +++ b/packages/firestore/src/remote/datastore.ts @@ -1,4 +1,3 @@ -import { WatchStreamListener, WriteStreamListener } from './persistent_stream'; /** * Copyright 2017 Google Inc. * @@ -24,7 +23,7 @@ import { Mutation, MutationResult } from '../model/mutation'; import { assert } from '../util/assert'; import { Code, FirestoreError } from '../util/error'; import { AsyncQueue } from '../util/async_queue'; - +import { WatchStreamListener, WriteStreamListener } from './persistent_stream'; import { Connection } from './connection'; import { PersistentListenStream, diff --git a/packages/firestore/src/remote/remote_event.ts b/packages/firestore/src/remote/remote_event.ts index 6663fb0ae21..a5e7a183fcb 100644 --- a/packages/firestore/src/remote/remote_event.ts +++ b/packages/firestore/src/remote/remote_event.ts @@ -16,8 +16,15 @@ import { SnapshotVersion } from '../core/snapshot_version'; import { ProtoByteString, TargetId } from '../core/types'; -import { DocumentKeySet, MaybeDocumentMap } from '../model/collections'; +import { + documentKeySet, + DocumentKeySet, + maybeDocumentMap, + MaybeDocumentMap, + targetIdSet +} from '../model/collections'; import { SortedSet } from '../util/sorted_set'; +import { emptyByteString } from '../platform/platform'; /** * An event from the RemoteStore. It is split into targetChanges (changes to the @@ -49,6 +56,32 @@ export class RemoteEvent { */ readonly resolvedLimboDocuments: DocumentKeySet ) {} + + /** + * HACK: Views require RemoteEvents in order to determine whether the view is + * CURRENT, but secondary tabs don't receive remote events. So this method is + * used to create a synthesized RemoteEvent that can be used to apply a + * CURRENT status change to a View, for queries executed in a different tab. + */ + // PORTING NOTE: Multi-tab only + static createSynthesizedRemoteEventForCurrentChange( + targetId: TargetId, + current: boolean + ): RemoteEvent { + const targetChanges = { + [targetId]: TargetChange.createSynthesizedTargetChangeForCurrentChange( + targetId, + current + ) + }; + return new RemoteEvent( + SnapshotVersion.MIN, + targetChanges, + targetIdSet(), + maybeDocumentMap(), + documentKeySet() + ); + } } /** @@ -90,4 +123,24 @@ export class TargetChange { */ readonly removedDocuments: DocumentKeySet ) {} + + /** + * HACK: Views require TargetChanges in order to determine whether the view is + * CURRENT, but secondary tabs don't receive remote events. So this method is + * used to create a synthesized TargetChanges that can be used to apply a + * CURRENT status change to a View, for queries executed in a different tab. + */ + // PORTING NOTE: Multi-tab only + static createSynthesizedTargetChangeForCurrentChange( + targetId: TargetId, + current: boolean + ): TargetChange { + return new TargetChange( + emptyByteString(), + current, + documentKeySet(), + documentKeySet(), + documentKeySet() + ); + } } diff --git a/packages/firestore/src/remote/remote_store.ts b/packages/firestore/src/remote/remote_store.ts index 8eff328d7d7..f1e4f5ac41a 100644 --- a/packages/firestore/src/remote/remote_store.ts +++ b/packages/firestore/src/remote/remote_store.ts @@ -51,6 +51,7 @@ import { import { OnlineStateTracker } from './online_state_tracker'; import { AsyncQueue } from '../util/async_queue'; import { DocumentKeySet } from '../model/collections'; +import { isPrimaryLeaseLostError } from '../local/indexeddb_persistence'; const LOG_TAG = 'RemoteStore'; @@ -107,12 +108,18 @@ export class RemoteStore implements TargetMetadataProvider { */ private listenTargets: { [targetId: number]: QueryData } = {}; - private networkEnabled = false; - private watchStream: PersistentListenStream; private writeStream: PersistentWriteStream; private watchChangeAggregator: WatchChangeAggregator = null; + /** + * Set to true by enableNetwork() and false by disableNetwork() and indicates + * the user-preferred network state. + */ + private networkEnabled = false; + + private isPrimary = false; + private onlineStateTracker: OnlineStateTracker; constructor( @@ -152,14 +159,15 @@ export class RemoteStore implements TargetMetadataProvider { * Starts up the remote store, creating streams, restoring state from * LocalStore, etc. */ - async start(): Promise { - await this.enableNetwork(); + start(): Promise { + return this.enableNetwork(); } /** Re-enables the network. Idempotent. */ async enableNetwork(): Promise { - if (!this.networkEnabled) { - this.networkEnabled = true; + this.networkEnabled = true; + + if (this.canUseNetwork()) { this.writeStream.lastStreamToken = await this.localStore.getLastStreamToken(); if (this.shouldStartWatchStream()) { @@ -178,6 +186,7 @@ export class RemoteStore implements TargetMetadataProvider { * enableNetwork(). */ async disableNetwork(): Promise { + this.networkEnabled = false; await this.disableNetworkInternal(); // Set the OnlineState to Offline so get()s return from cache, etc. @@ -185,28 +194,23 @@ export class RemoteStore implements TargetMetadataProvider { } private async disableNetworkInternal(): Promise { - if (this.networkEnabled) { - this.networkEnabled = false; + await this.writeStream.stop(); + await this.watchStream.stop(); - await this.writeStream.stop(); - await this.watchStream.stop(); - - if (this.writePipeline.length > 0) { - log.debug( - LOG_TAG, - `Stopping write stream with ${ - this.writePipeline.length - } pending writes` - ); - this.writePipeline = []; - } - - this.cleanUpWatchStreamState(); + if (this.writePipeline.length > 0) { + log.debug( + LOG_TAG, + `Stopping write stream with ${this.writePipeline.length} pending writes` + ); + this.writePipeline = []; } + + this.cleanUpWatchStreamState(); } async shutdown(): Promise { log.debug(LOG_TAG, 'RemoteStore shutting down.'); + this.networkEnabled = false; await this.disableNetworkInternal(); // Set the OnlineState to Unknown (rather than Offline) to avoid potentially @@ -299,9 +303,7 @@ export class RemoteStore implements TargetMetadataProvider { } private canUseNetwork(): boolean { - // TODO(mikelehen): This could take into account isPrimary when we merge - // with multitab. - return this.networkEnabled; + return this.isPrimary && this.networkEnabled; } private cleanUpWatchStreamState(): void { @@ -368,15 +370,13 @@ export class RemoteStore implements TargetMetadataProvider { this.watchChangeAggregator.handleTargetChange(watchChange); } - if ( - !snapshotVersion.isEqual(SnapshotVersion.MIN) && - snapshotVersion.compareTo( - this.localStore.getLastRemoteSnapshotVersion() - ) >= 0 - ) { - // We have received a target change with a global snapshot if the snapshot - // version is not equal to SnapshotVersion.MIN. - await this.raiseWatchSnapshot(snapshotVersion); + if (!snapshotVersion.isEqual(SnapshotVersion.MIN)) { + const lastRemoteSnapshotVersion = await this.localStore.getLastRemoteSnapshotVersion(); + if (snapshotVersion.compareTo(lastRemoteSnapshotVersion) >= 0) { + // We have received a target change with a global snapshot if the snapshot + // version is not equal to SnapshotVersion.MIN. + await this.raiseWatchSnapshot(snapshotVersion); + } } } @@ -501,7 +501,7 @@ export class RemoteStore implements TargetMetadataProvider { */ private canAddToWritePipeline(): boolean { return ( - this.networkEnabled && this.writePipeline.length < MAX_PENDING_WRITES + this.canUseNetwork() && this.writePipeline.length < MAX_PENDING_WRITES ); } @@ -555,7 +555,24 @@ export class RemoteStore implements TargetMetadataProvider { for (const batch of this.writePipeline) { this.writeStream.writeMutations(batch.mutations); } - }); + }) + .catch(err => this.ignoreIfPrimaryLeaseLoss(err)); + } + + /** + * Verifies the error thrown by an LocalStore operation. If a LocalStore + * operation fails because the primary lease has been taken by another client, + * we ignore the error. All other errors are re-thrown. + * + * @param err An error returned by a LocalStore operation. + * @return A Promise that resolves after we recovered, or the original error. + */ + private ignoreIfPrimaryLeaseLoss(err: FirestoreError): void { + if (isPrimaryLeaseLostError(err)) { + log.debug(LOG_TAG, 'Unexpectedly lost primary lease'); + } else { + throw err; + } } private onMutationResult( @@ -629,7 +646,9 @@ export class RemoteStore implements TargetMetadataProvider { ); this.writeStream.lastStreamToken = emptyByteString(); - return this.localStore.setLastStreamToken(emptyByteString()); + return this.localStore + .setLastStreamToken(emptyByteString()) + .catch(err => this.ignoreIfPrimaryLeaseLoss(err)); } else { // Some other error, don't reset stream token. Our stream logic will // just retry with exponential backoff. @@ -666,13 +685,28 @@ export class RemoteStore implements TargetMetadataProvider { async handleUserChange(user: User): Promise { log.debug(LOG_TAG, 'RemoteStore changing users: uid=', user.uid); - if (this.networkEnabled) { + if (this.canUseNetwork()) { // Tear down and re-create our network streams. This will ensure we get a fresh auth token // for the new user and re-fill the write pipeline with new mutations from the LocalStore // (since mutations are per-user). + this.networkEnabled = false; await this.disableNetworkInternal(); this.onlineStateTracker.set(OnlineState.Unknown); await this.enableNetwork(); } } + + /** + * Toggles the network state when the client gains or loses its primary lease. + */ + async applyPrimaryState(isPrimary: boolean): Promise { + this.isPrimary = isPrimary; + + if (isPrimary && this.networkEnabled) { + await this.enableNetwork(); + } else if (!isPrimary) { + await this.disableNetworkInternal(); + this.onlineStateTracker.set(OnlineState.Unknown); + } + } } diff --git a/packages/firestore/src/remote/remote_syncer.ts b/packages/firestore/src/remote/remote_syncer.ts index fdac5e3f5c9..4b673ce6562 100644 --- a/packages/firestore/src/remote/remote_syncer.ts +++ b/packages/firestore/src/remote/remote_syncer.ts @@ -21,7 +21,7 @@ import { RemoteEvent } from './remote_event'; import { DocumentKeySet } from '../model/collections'; /** - * A interface that describes the actions the RemoteStore needs to perform on + * An interface that describes the actions the RemoteStore needs to perform on * a cooperating synchronization engine. */ export interface RemoteSyncer { diff --git a/packages/firestore/src/util/async_queue.ts b/packages/firestore/src/util/async_queue.ts index a12acd0f3ef..19ece5b3cac 100644 --- a/packages/firestore/src/util/async_queue.ts +++ b/packages/firestore/src/util/async_queue.ts @@ -50,7 +50,13 @@ export enum TimerId { * OnlineState.Unknown to Offline after a set timeout, rather than waiting * indefinitely for success or failure. */ - OnlineStateTimeout = 'online_state_timeout' + OnlineStateTimeout = 'online_state_timeout', + + /** + * A timer used to update the client metadata in IndexedDb, which is used + * to determine the primary leaseholder. + */ + ClientMetadataRefresh = 'client_metadata_refresh' } /** @@ -249,6 +255,11 @@ export class AsyncQueue { ): CancelablePromise { this.verifyNotFailed(); + assert( + delayMs >= 0, + `Attempted to schedule an operation with a negative delay of ${delayMs}` + ); + // While not necessarily harmful, we currently don't expect to have multiple // ops with the same timer id in the queue, so defensively reject them. assert( diff --git a/packages/firestore/src/util/misc.ts b/packages/firestore/src/util/misc.ts index 24aed7c2eca..49b97aebf28 100644 --- a/packages/firestore/src/util/misc.ts +++ b/packages/firestore/src/util/misc.ts @@ -59,6 +59,22 @@ export function primitiveComparator(left: T, right: T): number { return 0; } +/** Implementation of min() that ignores null values. */ +export function min(...values: Array): number | null { + let min = null; + + for (const value of values) { + if (value !== null) { + if (min == null) { + min = value; + } else if (value < min) { + min = value; + } + } + } + + return min; +} /** Duck-typed interface for objects that have an isEqual() method. */ export interface Equatable { isEqual(other: T): boolean; @@ -92,34 +108,6 @@ export function arrayEquals(left: Array>, right: T[]): boolean { return true; } - -/** - * Returns the largest lexicographically smaller string of equal or smaller - * length. Returns an empty string if there is no such predecessor (if the input - * is empty). - * - * Strings returned from this method can be invalid UTF-16 but this is sufficent - * in use for indexeddb because that depends on lexicographical ordering but - * shouldn't be used elsewhere. - */ -export function immediatePredecessor(s: string): string { - // We can decrement the last character in the string and be done - // unless that character is 0 (0x0000), in which case we have to erase the - // last character. - const lastIndex = s.length - 1; - if (s.length === 0) { - // Special case the empty string. - return ''; - } else if (s.charAt(lastIndex) === '\0') { - return s.substring(0, lastIndex); - } else { - return ( - s.substring(0, lastIndex) + - String.fromCharCode(s.charCodeAt(lastIndex) - 1) - ); - } -} - /** * Returns the immediate lexicographically-following string. This is useful to * construct an inclusive range for indexeddb iterators. diff --git a/packages/firestore/src/util/obj.ts b/packages/firestore/src/util/obj.ts index c56ded1e85e..cc99ce18e56 100644 --- a/packages/firestore/src/util/obj.ts +++ b/packages/firestore/src/util/obj.ts @@ -39,6 +39,13 @@ export function size(obj: Dict): number { return count; } +/** Extracts the numeric indices from a dictionary. */ +export function indices(obj: { [numberKey: number]: V }): number[] { + return Object.keys(obj).map(key => { + return Number(key); + }); +} + /** Returns the given value if it's defined or the defaultValue otherwise. */ export function defaulted(value: V | undefined, defaultValue: V): V { return value !== undefined ? value : defaultValue; diff --git a/packages/firestore/src/util/sorted_set.ts b/packages/firestore/src/util/sorted_set.ts index 85fe2bd6ecc..1c7da829016 100644 --- a/packages/firestore/src/util/sorted_set.ts +++ b/packages/firestore/src/util/sorted_set.ts @@ -140,6 +140,14 @@ export class SortedSet { return true; } + toArray(): T[] { + const res: T[] = []; + this.forEach(targetId => { + res.push(targetId); + }); + return res; + } + toString(): string { const result: T[] = []; this.forEach(elem => result.push(elem)); diff --git a/packages/firestore/test/unit/core/event_manager.test.ts b/packages/firestore/test/unit/core/event_manager.test.ts index a8daf2a02b1..a5f73d47bfc 100644 --- a/packages/firestore/test/unit/core/event_manager.test.ts +++ b/packages/firestore/test/unit/core/event_manager.test.ts @@ -117,7 +117,7 @@ describe('EventManager', () => { const viewSnap1: any = { query: query1 }; // tslint:disable-next-line:no-any mock ViewSnapshot. const viewSnap2: any = { query: query2 }; - eventManager.onChange([viewSnap1, viewSnap2]); + eventManager.onWatchChange([viewSnap1, viewSnap2]); expect(eventOrder).to.deep.equal([ 'listenable1', @@ -126,7 +126,7 @@ describe('EventManager', () => { ]); }); - it('will forward applyOnlineStateChange calls', async () => { + it('will forward onOnlineStateChange calls', async () => { const query = Query.atPath(path('foo/bar')); const fakeListener1 = fakeQueryListener(query); const events: OnlineState[] = []; @@ -139,7 +139,7 @@ describe('EventManager', () => { await eventManager.listen(fakeListener1); expect(events).to.deep.equal([OnlineState.Unknown]); - eventManager.applyOnlineStateChange(OnlineState.Online); + eventManager.onOnlineStateChange(OnlineState.Online); expect(events).to.deep.equal([OnlineState.Unknown, OnlineState.Online]); }); }); @@ -233,7 +233,7 @@ describe('QueryListener', () => { const snap1 = applyDocChanges(view).snapshot!; const changes = view.computeDocChanges(documentUpdates()); - const snap2 = view.applyChanges(changes, ackTarget()).snapshot!; + const snap2 = view.applyChanges(changes, true, ackTarget()).snapshot!; eventListenable.onViewSnapshot(snap1); // no event eventListenable.onViewSnapshot(snap2); // empty event @@ -257,7 +257,7 @@ describe('QueryListener', () => { const snap1 = applyDocChanges(view, doc1).snapshot!; const changes = view.computeDocChanges(documentUpdates()); - const snap2 = view.applyChanges(changes, ackTarget(doc1)).snapshot!; + const snap2 = view.applyChanges(changes, true, ackTarget(doc1)).snapshot!; const snap3 = applyDocChanges(view, doc2).snapshot!; filteredListener.onViewSnapshot(snap1); // local event @@ -371,11 +371,12 @@ describe('QueryListener', () => { const view = new View(query, documentKeySet()); const changes1 = view.computeDocChanges(documentUpdates(doc1)); - const snap1 = view.applyChanges(changes1).snapshot!; + const snap1 = view.applyChanges(changes1, true).snapshot!; const changes2 = view.computeDocChanges(documentUpdates(doc2)); - const snap2 = view.applyChanges(changes2).snapshot!; + const snap2 = view.applyChanges(changes2, true).snapshot!; const changes3 = view.computeDocChanges(documentUpdates()); - const snap3 = view.applyChanges(changes3, ackTarget(doc1, doc2)).snapshot!; + const snap3 = view.applyChanges(changes3, true, ackTarget(doc1, doc2)) + .snapshot!; listener.applyOnlineStateChange(OnlineState.Online); // no event listener.onViewSnapshot(snap1); // no event @@ -411,9 +412,9 @@ describe('QueryListener', () => { const view = new View(query, documentKeySet()); const changes1 = view.computeDocChanges(documentUpdates(doc1)); - const snap1 = view.applyChanges(changes1).snapshot!; + const snap1 = view.applyChanges(changes1, true).snapshot!; const changes2 = view.computeDocChanges(documentUpdates(doc2)); - const snap2 = view.applyChanges(changes2).snapshot!; + const snap2 = view.applyChanges(changes2, true).snapshot!; listener.applyOnlineStateChange(OnlineState.Online); // no event listener.onViewSnapshot(snap1); // no event @@ -451,7 +452,7 @@ describe('QueryListener', () => { const view = new View(query, documentKeySet()); const changes1 = view.computeDocChanges(documentUpdates()); - const snap1 = view.applyChanges(changes1).snapshot!; + const snap1 = view.applyChanges(changes1, true).snapshot!; listener.applyOnlineStateChange(OnlineState.Online); // no event listener.onViewSnapshot(snap1); // no event @@ -477,7 +478,7 @@ describe('QueryListener', () => { const view = new View(query, documentKeySet()); const changes1 = view.computeDocChanges(documentUpdates()); - const snap1 = view.applyChanges(changes1).snapshot!; + const snap1 = view.applyChanges(changes1, true).snapshot!; listener.applyOnlineStateChange(OnlineState.Offline); listener.onViewSnapshot(snap1); diff --git a/packages/firestore/test/unit/core/target_id_generator.test.ts b/packages/firestore/test/unit/core/target_id_generator.test.ts index f26334d9c7a..78eb109b03a 100644 --- a/packages/firestore/test/unit/core/target_id_generator.test.ts +++ b/packages/firestore/test/unit/core/target_id_generator.test.ts @@ -18,26 +18,44 @@ import { expect } from 'chai'; import { TargetIdGenerator } from '../../../src/core/target_id_generator'; describe('TargetIdGenerator', () => { - it('can initialize with increment and "after value"', () => { - expect(new TargetIdGenerator(0).next()).to.equal(2); + it('can initialize with generator and seed', () => { + expect(new TargetIdGenerator(0, 2).next()).to.equal(2); expect(new TargetIdGenerator(1).next()).to.equal(1); + }); + + it('rejects invalid seeds', () => { + expect(() => new TargetIdGenerator(0, 1)).to.throw( + 'Cannot supply target ID from different generator ID' + ); + expect(() => new TargetIdGenerator(1).after(2)).to.throw( + 'Cannot supply target ID from different generator ID' + ); + }); - expect(new TargetIdGenerator(1, -1).next()).to.equal(1); - expect(new TargetIdGenerator(1, 2).next()).to.equal(3); - expect(new TargetIdGenerator(1, 4).next()).to.equal(5); - expect(new TargetIdGenerator(1, 23).next()).to.equal(25); + it('rejects invalid generator IDs', () => { + expect(() => new TargetIdGenerator(3)).to.throw( + ' Generator ID 3 contains more than 1 reserved bits' + ); }); it('can increments ids', () => { - const generator = new TargetIdGenerator(1, 46); - expect(generator.next()).to.equal(47); + const generator = new TargetIdGenerator(1); + expect(generator.after(45)).to.equal(47); expect(generator.next()).to.equal(49); expect(generator.next()).to.equal(51); expect(generator.next()).to.equal(53); }); - it('can return correct generator for local store and sync engine', () => { - expect(TargetIdGenerator.forLocalStore().next()).to.equal(2); + it('can return correct generator for query cache and sync engine', () => { + expect(TargetIdGenerator.forQueryCache().next()).to.equal(2); expect(TargetIdGenerator.forSyncEngine().next()).to.equal(1); }); + + it('can return next ids', () => { + expect(new TargetIdGenerator(1).next()).to.equal(1); + expect(new TargetIdGenerator(1).after(-1)).to.equal(1); + expect(new TargetIdGenerator(1).after(1)).to.equal(3); + expect(new TargetIdGenerator(1).after(3)).to.equal(5); + expect(new TargetIdGenerator(1).after(23)).to.equal(25); + }); }); diff --git a/packages/firestore/test/unit/core/view.test.ts b/packages/firestore/test/unit/core/view.test.ts index 1f9a73371a9..1ff43f4c513 100644 --- a/packages/firestore/test/unit/core/view.test.ts +++ b/packages/firestore/test/unit/core/view.test.ts @@ -45,8 +45,11 @@ describe('View', () => { const doc3 = doc('rooms/other/messages/1', 0, { text: 'msg3' }); const changes = view.computeDocChanges(documentUpdates(doc1, doc2, doc3)); - const snapshot = view.applyChanges(changes, ackTarget(doc1, doc2, doc3)) - .snapshot!; + const snapshot = view.applyChanges( + changes, + true, + ackTarget(doc1, doc2, doc3) + ).snapshot!; expect(snapshot.query).to.deep.equal(query); expect(documentSetAsArray(snapshot.docs)).to.deep.equal([doc1, doc2]); @@ -73,7 +76,7 @@ describe('View', () => { // delete doc2, add doc3 const changes = view.computeDocChanges(documentUpdates(doc2.key, doc3)); - const snapshot = view.applyChanges(changes, ackTarget(doc1, doc3)) + const snapshot = view.applyChanges(changes, true, ackTarget(doc1, doc3)) .snapshot!; expect(snapshot.query).to.deep.equal(query); @@ -182,8 +185,11 @@ describe('View', () => { // add doc2, which should push out doc3 const changes = view.computeDocChanges(documentUpdates(doc2)); - const snapshot = view.applyChanges(changes, ackTarget(doc1, doc2, doc3)) - .snapshot!; + const snapshot = view.applyChanges( + changes, + true, + ackTarget(doc1, doc2, doc3) + ).snapshot!; expect(snapshot.query).to.deep.equal(query); expect(documentSetAsArray(snapshot.docs)).to.deep.equal([doc1, doc2]); @@ -224,6 +230,7 @@ describe('View', () => { ); const snapshot = view.applyChanges( changes, + true, ackTarget(doc1, doc2, doc3, doc4) ).snapshot!; @@ -245,17 +252,18 @@ describe('View', () => { const view = new View(query, documentKeySet()); let changes = view.computeDocChanges(documentUpdates(doc1)); - let viewChange = view.applyChanges(changes); + let viewChange = view.applyChanges(changes, true); expect(viewChange.limboChanges).to.deep.equal([]); changes = view.computeDocChanges(documentUpdates()); - viewChange = view.applyChanges(changes, ackTarget()); + viewChange = view.applyChanges(changes, true, ackTarget()); expect(viewChange.limboChanges).to.deep.equal( limboChanges({ added: [doc1] }) ); viewChange = view.applyChanges( changes, + true, updateMapping(version(0), [doc1], [], [], /* current= */ true) ); expect(viewChange.limboChanges).to.deep.equal( @@ -265,6 +273,7 @@ describe('View', () => { changes = view.computeDocChanges(documentUpdates(doc2)); viewChange = view.applyChanges( changes, + true, updateMapping(version(0), [doc2], [], [], /* current= */ true) ); expect(viewChange.limboChanges).to.deep.equal([]); @@ -291,7 +300,7 @@ describe('View', () => { const view = new View(query, keySet(doc1.key, doc2.key)); const changes = view.computeDocChanges(documentUpdates()); - const change = view.applyChanges(changes, ackTarget()); + const change = view.applyChanges(changes, true, ackTarget()); expect(change.limboChanges).to.deep.equal([]); }); @@ -306,7 +315,7 @@ describe('View', () => { expect(changes.documentSet.size).to.equal(2); expect(changes.needsRefill).to.equal(false); expect(changes.changeSet.getChanges().length).to.equal(2); - view.applyChanges(changes); + view.applyChanges(changes, true); // Remove one of the docs. changes = view.computeDocChanges(documentUpdates(doc1.key)); @@ -318,7 +327,7 @@ describe('View', () => { expect(changes.documentSet.size).to.equal(1); expect(changes.needsRefill).to.equal(false); expect(changes.changeSet.getChanges().length).to.equal(1); - view.applyChanges(changes); + view.applyChanges(changes, true); }); it('returns needsRefill on reorder in limit query', () => { @@ -335,7 +344,7 @@ describe('View', () => { expect(changes.documentSet.size).to.equal(2); expect(changes.needsRefill).to.equal(false); expect(changes.changeSet.getChanges().length).to.equal(2); - view.applyChanges(changes); + view.applyChanges(changes, true); // Move one of the docs. doc2 = doc(doc2.key.toString(), 1, { order: 2000 }); @@ -351,7 +360,7 @@ describe('View', () => { expect(changes.documentSet.size).to.equal(2); expect(changes.needsRefill).to.equal(false); expect(changes.changeSet.getChanges().length).to.equal(2); - view.applyChanges(changes); + view.applyChanges(changes, true); }); it("doesn't need refill on reorder within limit", () => { @@ -372,7 +381,7 @@ describe('View', () => { expect(changes.documentSet.size).to.equal(3); expect(changes.needsRefill).to.equal(false); expect(changes.changeSet.getChanges().length).to.equal(3); - view.applyChanges(changes); + view.applyChanges(changes, true); // Move one of the docs. doc1 = doc(doc1.key.toString(), 1, { order: 3 }); @@ -380,7 +389,7 @@ describe('View', () => { expect(changes.documentSet.size).to.equal(3); expect(changes.needsRefill).to.equal(false); expect(changes.changeSet.getChanges().length).to.equal(1); - view.applyChanges(changes); + view.applyChanges(changes, true); }); it("doesn't need refill on reorder after limit query", () => { @@ -401,7 +410,7 @@ describe('View', () => { expect(changes.documentSet.size).to.equal(3); expect(changes.needsRefill).to.equal(false); expect(changes.changeSet.getChanges().length).to.equal(3); - view.applyChanges(changes); + view.applyChanges(changes, true); // Move one of the docs. doc4 = doc(doc4.key.toString(), 1, { order: 6 }); @@ -409,7 +418,7 @@ describe('View', () => { expect(changes.documentSet.size).to.equal(3); expect(changes.needsRefill).to.equal(false); expect(changes.changeSet.getChanges().length).to.equal(0); - view.applyChanges(changes); + view.applyChanges(changes, true); }); it("doesn't need refill for additions after the limit", () => { @@ -423,7 +432,7 @@ describe('View', () => { expect(changes.documentSet.size).to.equal(2); expect(changes.needsRefill).to.equal(false); expect(changes.changeSet.getChanges().length).to.equal(2); - view.applyChanges(changes); + view.applyChanges(changes, true); // Add a doc that is past the limit. const doc3 = doc('rooms/eros/msgs/2', 0, {}); @@ -431,7 +440,7 @@ describe('View', () => { expect(changes.documentSet.size).to.equal(2); expect(changes.needsRefill).to.equal(false); expect(changes.changeSet.getChanges().length).to.equal(0); - view.applyChanges(changes); + view.applyChanges(changes, true); }); it("doesn't need refill for deletions when not near the limit", () => { @@ -444,14 +453,14 @@ describe('View', () => { expect(changes.documentSet.size).to.equal(2); expect(changes.needsRefill).to.equal(false); expect(changes.changeSet.getChanges().length).to.equal(2); - view.applyChanges(changes); + view.applyChanges(changes, true); // Remove one of the docs. changes = view.computeDocChanges(documentUpdates(doc2.key)); expect(changes.documentSet.size).to.equal(1); expect(changes.needsRefill).to.equal(false); expect(changes.changeSet.getChanges().length).to.equal(1); - view.applyChanges(changes); + view.applyChanges(changes, true); }); it('handles applying irrelevant docs', () => { @@ -465,7 +474,7 @@ describe('View', () => { expect(changes.documentSet.size).to.equal(2); expect(changes.needsRefill).to.equal(false); expect(changes.changeSet.getChanges().length).to.equal(2); - view.applyChanges(changes); + view.applyChanges(changes, true); // Remove a doc that isn't even in the results. const doc3 = doc('rooms/eros/msgs/2', 0, {}); @@ -473,7 +482,7 @@ describe('View', () => { expect(changes.documentSet.size).to.equal(2); expect(changes.needsRefill).to.equal(false); expect(changes.changeSet.getChanges().length).to.equal(0); - view.applyChanges(changes); + view.applyChanges(changes, true); }); it('computes mutatedDocKeys', () => { @@ -483,7 +492,7 @@ describe('View', () => { const view = new View(query, documentKeySet()); // Start with a full view. let changes = view.computeDocChanges(documentUpdates(doc1, doc2)); - view.applyChanges(changes); + view.applyChanges(changes, true); expect(changes.mutatedKeys).to.deep.equal(keySet()); const doc3 = doc('rooms/eros/msgs/2', 0, {}, { hasLocalMutations: true }); @@ -501,12 +510,12 @@ describe('View', () => { const view = new View(query, documentKeySet()); // Start with a full view. let changes = view.computeDocChanges(documentUpdates(doc1, doc2)); - view.applyChanges(changes); + view.applyChanges(changes, true); expect(changes.mutatedKeys).to.deep.equal(keySet(doc2.key)); const doc2prime = doc('rooms/eros/msgs/1', 0, {}); changes = view.computeDocChanges(documentUpdates(doc2prime)); - view.applyChanges(changes); + view.applyChanges(changes, true); expect(changes.mutatedKeys).to.deep.equal(keySet()); } ); @@ -518,7 +527,7 @@ describe('View', () => { const view = new View(query, documentKeySet()); // Start with a full view. let changes = view.computeDocChanges(documentUpdates(doc1, doc2)); - view.applyChanges(changes); + view.applyChanges(changes, true); expect(changes.mutatedKeys).to.deep.equal(keySet(doc2.key)); const doc3 = doc('rooms/eros/msgs/3', 0, {}); diff --git a/packages/firestore/test/unit/local/indexeddb_schema.test.ts b/packages/firestore/test/unit/local/indexeddb_schema.test.ts index 3767ce48fbc..50668cab56a 100644 --- a/packages/firestore/test/unit/local/indexeddb_schema.test.ts +++ b/packages/firestore/test/unit/local/indexeddb_schema.test.ts @@ -14,23 +14,39 @@ * limitations under the License. */ +// TODO(multitab): Rename this file to `indexeddb_persistence.test.ts`. + import { expect } from 'chai'; import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; import { - ALL_STORES, createOrUpgradeDb, DbMutationBatch, DbMutationBatchKey, + DbOwner, + DbOwnerKey, DbTarget, DbTargetGlobal, DbTargetGlobalKey, DbTargetKey, - DbTimestamp + DbTimestamp, + SCHEMA_VERSION, + V1_STORES, + V3_STORES, + V4_STORES } from '../../../src/local/indexeddb_schema'; import { SimpleDb, SimpleDbTransaction } from '../../../src/local/simple_db'; +import { PersistencePromise } from '../../../src/local/persistence_promise'; +import { ClientId } from '../../../src/local/shared_client_state'; +import { DatabaseId } from '../../../src/core/database_info'; +import { JsonProtoSerializer } from '../../../src/remote/serializer'; +import { PlatformSupport } from '../../../src/platform/platform'; +import { AsyncQueue } from '../../../src/util/async_queue'; +import { SharedFakeWebStorage, TestPlatform } from '../../util/test_platform'; import { SnapshotVersion } from '../../../src/core/snapshot_version'; -const INDEXEDDB_TEST_DATABASE = 'schemaTest'; +const INDEXEDDB_TEST_DATABASE_PREFIX = 'schemaTest/'; +const INDEXEDDB_TEST_DATABASE = + INDEXEDDB_TEST_DATABASE_PREFIX + IndexedDbPersistence.MAIN_DATABASE; function withDb( schemaVersion, @@ -63,6 +79,36 @@ function withDb( }); } +async function withPersistence( + clientId: ClientId, + fn: ( + persistence: IndexedDbPersistence, + platform: TestPlatform, + queue: AsyncQueue + ) => Promise +): Promise { + const partition = new DatabaseId('project'); + const serializer = new JsonProtoSerializer(partition, { + useProto3Json: true + }); + + const queue = new AsyncQueue(); + const platform = new TestPlatform( + PlatformSupport.getPlatform(), + new SharedFakeWebStorage() + ); + const persistence = new IndexedDbPersistence( + INDEXEDDB_TEST_DATABASE_PREFIX, + clientId, + platform, + queue, + serializer + ); + + await fn(persistence, platform, queue); + await persistence.shutdown(); +} + function getAllObjectStores(db: IDBDatabase): string[] { const objectStores: string[] = []; for (let i = 0; i < db.objectStoreNames.length; ++i) { @@ -83,11 +129,10 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => { after(() => SimpleDb.delete(INDEXEDDB_TEST_DATABASE)); it('can install schema version 1', () => { - return withDb(1, db => { + return withDb(1, async db => { expect(db.version).to.equal(1); // Version 1 adds all of the stores so far. - expect(getAllObjectStores(db)).to.have.members(ALL_STORES); - return Promise.resolve(); + expect(getAllObjectStores(db)).to.have.members(V1_STORES); }); }); @@ -138,7 +183,7 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => { }).then(() => { return withDb(3, db => { expect(db.version).to.equal(3); - expect(getAllObjectStores(db)).to.have.members(ALL_STORES); + expect(getAllObjectStores(db)).to.have.members(V3_STORES); const sdb = new SimpleDb(db); return sdb.runTransaction( @@ -167,7 +212,7 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => { const expected = JSON.parse(JSON.stringify(resetTargetGlobal)); expect(targetGlobalEntry).to.deep.equal(expected); }) - .next(() => mutations.get([userId, batchId])) + .next(() => mutations.get(batchId)) .next(mutation => { // Mutations should be unaffected. expect(mutation.userId).to.equal(userId); @@ -178,4 +223,227 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => { }); }); }); + + it('can upgrade from schema version 3 to 4', () => { + const testWrite = { delete: 'foo' }; + const testMutations = [ + { + userId: 'foo', + batchId: 0, + localWriteTime: 1337, + mutations: [] + }, + { + userId: 'foo', + batchId: 1, + localWriteTime: 1337, + mutations: [testWrite] + }, + { + userId: 'foo', + batchId: 42, + localWriteTime: 1337, + mutations: [testWrite, testWrite] + } + ]; + + return withDb(3, db => { + const sdb = new SimpleDb(db); + return sdb.runTransaction('readwrite', [DbMutationBatch.store], txn => { + const store = txn.store(DbMutationBatch.store); + let p = PersistencePromise.resolve(); + for (const testMutation of testMutations) { + p = p.next(() => store.put(testMutation)); + } + return p; + }); + }).then(() => + withDb(4, db => { + expect(db.version).to.be.equal(4); + expect(getAllObjectStores(db)).to.have.members(V4_STORES); + + const sdb = new SimpleDb(db); + return sdb.runTransaction('readwrite', [DbMutationBatch.store], txn => { + const store = txn.store( + DbMutationBatch.store + ); + let p = PersistencePromise.resolve(); + for (const testMutation of testMutations) { + p = p.next(() => + store.get(testMutation.batchId).next(mutationBatch => { + expect(mutationBatch).to.deep.equal(testMutation); + }) + ); + } + p = p.next(() => { + store + .add({} as any) // tslint:disable-line:no-any + .next(batchId => { + expect(batchId).to.equal(43); + }); + }); + return p; + }); + }) + ); + }); +}); + +describe('IndexedDb: canActAsPrimary', () => { + if (!IndexedDbPersistence.isAvailable()) { + console.warn('No IndexedDB. Skipping canActAsPrimary() tests.'); + return; + } + + async function clearOwner(): Promise { + const simpleDb = await SimpleDb.openOrCreate( + INDEXEDDB_TEST_DATABASE, + SCHEMA_VERSION, + createOrUpgradeDb + ); + await simpleDb.runTransaction('readwrite', [DbOwner.store], txn => { + const ownerStore = txn.store(DbOwner.store); + return ownerStore.delete('owner'); + }); + simpleDb.close(); + } + + beforeEach(() => { + return SimpleDb.delete(INDEXEDDB_TEST_DATABASE); + }); + + after(() => SimpleDb.delete(INDEXEDDB_TEST_DATABASE)); + + const visible: VisibilityState = 'visible'; + const hidden: VisibilityState = 'hidden'; + const networkEnabled = true; + const networkDisabled = false; + const primary = true; + const secondary = false; + + type ExpectedPrimaryStateTestCase = [ + boolean, + VisibilityState, + boolean, + VisibilityState, + boolean + ]; + + const testCases: ExpectedPrimaryStateTestCase[] = [ + [networkDisabled, hidden, networkDisabled, hidden, primary], + [networkDisabled, hidden, networkDisabled, visible, primary], + [networkDisabled, hidden, networkEnabled, hidden, primary], + [networkDisabled, hidden, networkEnabled, visible, primary], + [networkDisabled, visible, networkDisabled, hidden, secondary], + [networkDisabled, visible, networkDisabled, visible, primary], + [networkDisabled, visible, networkEnabled, hidden, primary], + [networkDisabled, visible, networkEnabled, visible, primary], + [networkEnabled, hidden, networkDisabled, hidden, secondary], + [networkEnabled, hidden, networkDisabled, visible, secondary], + [networkEnabled, hidden, networkEnabled, hidden, primary], + [networkEnabled, hidden, networkEnabled, visible, primary], + [networkEnabled, visible, networkDisabled, hidden, secondary], + [networkEnabled, visible, networkDisabled, visible, secondary], + [networkEnabled, visible, networkEnabled, hidden, secondary], + [networkEnabled, visible, networkEnabled, visible, primary] + ]; + + for (const testCase of testCases) { + const [ + thatNetwork, + thatVisibility, + thisNetwork, + thisVisibility, + primaryState + ] = testCase; + const testName = `is ${ + primaryState ? 'eligible' : 'not eligible' + } when client is ${ + thisNetwork ? 'online' : 'offline' + } and ${thisVisibility} and other client is ${ + thatNetwork ? 'online' : 'offline' + } and ${thatVisibility}`; + + it(testName, () => { + return withPersistence( + 'thatClient', + async (thatPersistence, thatPlatform, thatQueue) => { + await thatPersistence.start(); + thatPlatform.raiseVisibilityEvent(thatVisibility); + thatPersistence.setNetworkEnabled(thatNetwork); + await thatQueue.drain(); + + // Clear the current primary holder, since our logic will not revoke + // the lease until it expires. + await clearOwner(); + + await withPersistence( + 'thisClient', + async (thisPersistence, thisPlatform, thisQueue) => { + await thisPersistence.start(); + thisPlatform.raiseVisibilityEvent(thisVisibility); + thisPersistence.setNetworkEnabled(thisNetwork); + await thisQueue.drain(); + + let isPrimary: boolean; + await thisPersistence.setPrimaryStateListener( + async primaryState => { + isPrimary = primaryState; + } + ); + expect(isPrimary).to.eq(primaryState); + } + ); + } + ); + }); + } + + it('is eligible when only client', () => { + return withPersistence('clientA', async (persistence, platform, queue) => { + await persistence.start(); + platform.raiseVisibilityEvent('hidden'); + persistence.setNetworkEnabled(false); + await queue.drain(); + + let isPrimary: boolean; + await persistence.setPrimaryStateListener(async primaryState => { + isPrimary = primaryState; + }); + expect(isPrimary).to.be.true; + }); + }); +}); + +describe('IndexedDb: allowTabSynchronization', () => { + if (!IndexedDbPersistence.isAvailable()) { + console.warn('No IndexedDB. Skipping allowTabSynchronization tests.'); + return; + } + + beforeEach(() => SimpleDb.delete(INDEXEDDB_TEST_DATABASE)); + + after(() => SimpleDb.delete(INDEXEDDB_TEST_DATABASE)); + + it('rejects access when synchronization is disabled', () => { + return withPersistence('clientA', async db1 => { + await db1.start(/*synchronizeTabs=*/ false); + await withPersistence('clientB', async db2 => { + await expect( + db2.start(/*synchronizeTabs=*/ false) + ).to.eventually.be.rejectedWith( + 'Another tab has exclusive access to the persistence layer.' + ); + }); + }); + }); + + it('grants access when synchronization is enabled', async () => { + return withPersistence('clientA', async db1 => { + await db1.start(/*synchronizeTabs=*/ true); + await withPersistence('clientB', async db2 => { + await db2.start(/*synchronizeTabs=*/ true); + }); + }); + }); }); diff --git a/packages/firestore/test/unit/local/local_store.test.ts b/packages/firestore/test/unit/local/local_store.test.ts index 7c00d821dd2..45f570ed084 100644 --- a/packages/firestore/test/unit/local/local_store.test.ts +++ b/packages/firestore/test/unit/local/local_store.test.ts @@ -67,6 +67,7 @@ import { } from '../../util/helpers'; import * as persistenceHelpers from './persistence_test_helpers'; +import { MemorySharedClientState } from '../../../src/local/shared_client_state'; class LocalStoreTester { private promiseChain: Promise = Promise.resolve(); @@ -172,7 +173,10 @@ class LocalStoreTester { afterReleasingQuery(query: Query): LocalStoreTester { this.promiseChain = this.promiseChain.then(() => { - return this.localStore.releaseQuery(query); + return this.localStore.releaseQuery( + query, + /*keepPersistedQueryData=*/ false + ); }); return this; } @@ -276,16 +280,15 @@ function genericLocalStoreTests( let persistence: Persistence; let localStore: LocalStore; - beforeEach(() => { - return getPersistence().then(p => { - persistence = p; - localStore = new LocalStore( - persistence, - User.UNAUTHENTICATED, - new EagerGarbageCollector() - ); - return localStore.start(); - }); + beforeEach(async () => { + persistence = await getPersistence(); + localStore = new LocalStore( + persistence, + User.UNAUTHENTICATED, + new EagerGarbageCollector(), + new MemorySharedClientState() + ); + return localStore.start(); }); afterEach(() => persistence.shutdown(/* deleteData= */ true)); @@ -298,7 +301,8 @@ function genericLocalStoreTests( localStore = new LocalStore( persistence, User.UNAUTHENTICATED, - new NoOpGarbageCollector() + new NoOpGarbageCollector(), + new MemorySharedClientState() ); return localStore.start(); } @@ -890,7 +894,7 @@ function genericLocalStoreTests( await localStore.applyRemoteEvent(remoteEvent); // Stop listening so that the query should become inactive (but persistent) - await localStore.releaseQuery(query); + await localStore.releaseQuery(query, /*keepPersistedQueryData=*/ false); // Should come back with the same resume token const queryData2 = await localStore.allocateQuery(query); @@ -931,7 +935,7 @@ function genericLocalStoreTests( await localStore.applyRemoteEvent(remoteEvent2); // Stop listening so that the query should become inactive (but persistent) - await localStore.releaseQuery(query); + await localStore.releaseQuery(query, /*keepPersistedQueryData=*/ false); // Should come back with the same resume token const queryData2 = await localStore.allocateQuery(query); diff --git a/packages/firestore/test/unit/local/mutation_queue.test.ts b/packages/firestore/test/unit/local/mutation_queue.test.ts index 4da31bbcc3e..ec6d91f27a0 100644 --- a/packages/firestore/test/unit/local/mutation_queue.test.ts +++ b/packages/firestore/test/unit/local/mutation_queue.test.ts @@ -17,11 +17,8 @@ import { expect } from 'chai'; import { User } from '../../../src/auth/user'; import { Query } from '../../../src/core/query'; -import { BatchId } from '../../../src/core/types'; import { EagerGarbageCollector } from '../../../src/local/eager_garbage_collector'; -import { IndexedDbMutationQueue } from '../../../src/local/indexeddb_mutation_queue'; import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; -import { DbMutationBatch } from '../../../src/local/indexeddb_schema'; import { Persistence } from '../../../src/local/persistence'; import { documentKeySet } from '../../../src/model/collections'; import { @@ -67,52 +64,6 @@ describe('IndexedDbMutationQueue', () => { }); genericMutationQueueTests(); - - describe('loadNextBatchIdFromDb', () => { - function loadNextBatchId(): Promise { - return persistence.runTransaction('loadNextBatchIdFromDb', txn => { - return IndexedDbMutationQueue.loadNextBatchIdFromDb(txn).next( - batchId => { - return batchId; - } - ); - }); - } - - function addDummyBatch(userId: string, batchId: BatchId): Promise { - return persistence.runTransaction('addDummyBatch', transaction => { - const store = IndexedDbPersistence.getStore< - [string, number], - DbMutationBatch - >(transaction, DbMutationBatch.store); - const localWriteTime = Date.now(); - return store.put( - new DbMutationBatch(userId, batchId, localWriteTime, []) - ); - }); - } - - it('returns zero when no mutations.', async () => { - const batchId = await loadNextBatchId(); - expect(batchId).to.equal(0); - }); - - it('finds next id after single mutation batch', async () => { - await addDummyBatch('foo', 6); - expect(await loadNextBatchId()).to.equal(7); - }); - - it('finds max across users', async () => { - await addDummyBatch('fo', 5); - await addDummyBatch('food', 3); - - await addDummyBatch('foo', 6); - await addDummyBatch('foo', 2); - await addDummyBatch('foo', 1); - - expect(await loadNextBatchId()).to.equal(7); - }); - }); }); /** @@ -265,7 +216,10 @@ function genericMutationQueueTests(): void { it('getHighestAcknowledgedBatchId() never exceeds getNextBatchId()', async () => { const batch1 = await addMutationBatch(); + expect(batch1.batchId).to.equal(1); const batch2 = await addMutationBatch(); + expect(batch2.batchId).to.equal(2); + await mutationQueue.acknowledgeBatch(batch1, emptyByteString()); await mutationQueue.acknowledgeBatch(batch2, emptyByteString()); expect(await mutationQueue.getHighestAcknowledgedBatchId()).to.equal( @@ -284,19 +238,11 @@ function genericMutationQueueTests(): void { ); await mutationQueue.start(); - // Verify that on restart with an empty queue, nextBatchID falls to a - // lower value. - expect(await mutationQueue.getNextBatchId()).to.be.lessThan(batch2.batchId); - - // As a result highestAcknowledgedBatchID must also reset lower. - expect(await mutationQueue.getHighestAcknowledgedBatchId()).to.equal( - BATCHID_UNKNOWN - ); - - // The mutation queue will reset the next batchID after all mutations - // are removed so adding another mutation will cause a collision. - const newBatch = await addMutationBatch(); - expect(newBatch.batchId).to.equal(batch1.batchId); + // PORTING NOTE: On the Web, the mutation queue does not reset the next + // batchID after all mutations are removed. Adding another mutation will + // never cause a collision. + const batch3 = await addMutationBatch(); + expect(batch3.batchId).to.equal(3); // Restart the queue with one unacknowledged batch in it. mutationQueue = new TestMutationQueue( @@ -305,11 +251,12 @@ function genericMutationQueueTests(): void { ); await mutationQueue.start(); - expect(await mutationQueue.getNextBatchId()).to.equal(newBatch.batchId + 1); + const batch4 = await addMutationBatch(); + expect(batch4.batchId).to.equal(4); - // highestAcknowledgedBatchID must still be BATCHID_UNKNOWN. + // highestAcknowledgedBatchID must still be batch2. expect(await mutationQueue.getHighestAcknowledgedBatchId()).to.equal( - BATCHID_UNKNOWN + batch2.batchId ); }); diff --git a/packages/firestore/test/unit/local/persistence_test_helpers.ts b/packages/firestore/test/unit/local/persistence_test_helpers.ts index f9acd7790de..9c90707d5f3 100644 --- a/packages/firestore/test/unit/local/persistence_test_helpers.ts +++ b/packages/firestore/test/unit/local/persistence_test_helpers.ts @@ -19,28 +19,137 @@ import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; import { MemoryPersistence } from '../../../src/local/memory_persistence'; import { SimpleDb } from '../../../src/local/simple_db'; import { JsonProtoSerializer } from '../../../src/remote/serializer'; +import { + WebStorageSharedClientState, + ClientId +} from '../../../src/local/shared_client_state'; +import { + BatchId, + MutationBatchState, + OnlineState, + TargetId +} from '../../../src/core/types'; +import { AsyncQueue } from '../../../src/util/async_queue'; +import { User } from '../../../src/auth/user'; +import { + QueryTargetState, + SharedClientStateSyncer +} from '../../../src/local/shared_client_state_syncer'; +import { FirestoreError } from '../../../src/util/error'; +import { AutoId } from '../../../src/util/misc'; +import { PlatformSupport } from '../../../src/platform/platform'; + +/** The persistence prefix used for testing in IndexedBD and LocalStorage. */ +export const TEST_PERSISTENCE_PREFIX = + 'firestore/[DEFAULT]/PersistenceTestHelpers'; + +/** The prefix used by the keys that Firestore writes to Local Storage. */ +const LOCAL_STORAGE_PREFIX = 'firestore_'; /** * Creates and starts an IndexedDbPersistence instance for testing, destroying * any previous contents if they existed. */ -export async function testIndexedDbPersistence(): Promise< - IndexedDbPersistence -> { - const prefix = 'PersistenceTestHelpers/'; +export async function testIndexedDbPersistence( + synchronizeTabs?: boolean +): Promise { + const queue = new AsyncQueue(); + const clientId = AutoId.newId(); + const prefix = `${TEST_PERSISTENCE_PREFIX}/`; await SimpleDb.delete(prefix + IndexedDbPersistence.MAIN_DATABASE); const partition = new DatabaseId('project'); const serializer = new JsonProtoSerializer(partition, { useProto3Json: true }); - const persistence = new IndexedDbPersistence(prefix, serializer); - await persistence.start(); + const platform = PlatformSupport.getPlatform(); + const persistence = new IndexedDbPersistence( + prefix, + clientId, + platform, + queue, + serializer + ); + await persistence.start(synchronizeTabs); return persistence; } /** Creates and starts a MemoryPersistence instance for testing. */ export async function testMemoryPersistence(): Promise { - const persistence = new MemoryPersistence(); + const persistence = new MemoryPersistence(AutoId.newId()); await persistence.start(); return persistence; } + +class NoOpSharedClientStateSyncer implements SharedClientStateSyncer { + constructor(private readonly activeClients: ClientId[]) {} + async applyBatchState( + batchId: BatchId, + state: MutationBatchState, + error?: FirestoreError + ): Promise {} + async applySuccessfulWrite(batchId: BatchId): Promise {} + async rejectFailedWrite( + batchId: BatchId, + err: FirestoreError + ): Promise {} + async getActiveClients(): Promise { + return this.activeClients; + } + async applyTargetState( + targetId: TargetId, + state: QueryTargetState, + error?: FirestoreError + ): Promise {} + async applyActiveTargetsChange( + added: TargetId[], + removed: TargetId[] + ): Promise {} + applyOnlineStateChange(onlineState: OnlineState): void {} +} +/** + * Populates Web Storage with instance data from a pre-existing client. + */ +export async function populateWebStorage( + user: User, + existingClientId: ClientId, + existingMutationBatchIds: BatchId[], + existingQueryTargetIds: TargetId[] +): Promise { + // HACK: Create a secondary client state to seed data into LocalStorage. + // NOTE: We don't call shutdown() on it because that would delete the data. + const secondaryClientState = new WebStorageSharedClientState( + new AsyncQueue(), + PlatformSupport.getPlatform(), + TEST_PERSISTENCE_PREFIX, + existingClientId, + user + ); + + secondaryClientState.syncEngine = new NoOpSharedClientStateSyncer([ + existingClientId + ]); + secondaryClientState.onlineStateHandler = () => {}; + await secondaryClientState.start(); + + for (const batchId of existingMutationBatchIds) { + secondaryClientState.addLocalPendingMutation(batchId); + } + + for (const targetId of existingQueryTargetIds) { + secondaryClientState.addLocalQueryTarget(targetId); + } +} + +/** + * Removes Firestore data (by prefix match) from Local Storage. + */ +export function clearWebStorage(): void { + for (let i = 0; ; ++i) { + const key = window.localStorage.key(i); + if (key === null) { + break; + } else if (key.startsWith(LOCAL_STORAGE_PREFIX)) { + window.localStorage.removeItem(key); + } + } +} diff --git a/packages/firestore/test/unit/local/query_cache.test.ts b/packages/firestore/test/unit/local/query_cache.test.ts index 09fedceaa05..6fa874c1073 100644 --- a/packages/firestore/test/unit/local/query_cache.test.ts +++ b/packages/firestore/test/unit/local/query_cache.test.ts @@ -127,11 +127,11 @@ function genericQueryCacheTests( // equal canonicalIDs. expect(await cache.getQueryData(q2)).to.equal(null); expect(await cache.getQueryData(q1)).to.deep.equal(data1); - expect(cache.count()).to.equal(1); + expect(await cache.getQueryCount()).to.equal(1); const data2 = testQueryData(q2, 2, 1); await cache.addQueryData(data2); - expect(cache.count()).to.equal(2); + expect(await cache.getQueryCount()).to.equal(2); expect(await cache.getQueryData(q1)).to.deep.equal(data1); expect(await cache.getQueryData(q2)).to.deep.equal(data2); @@ -139,12 +139,12 @@ function genericQueryCacheTests( await cache.removeQueryData(data1); expect(await cache.getQueryData(q1)).to.equal(null); expect(await cache.getQueryData(q2)).to.deep.equal(data2); - expect(cache.count()).to.equal(1); + expect(await cache.getQueryCount()).to.equal(1); await cache.removeQueryData(data2); expect(await cache.getQueryData(q1)).to.equal(null); expect(await cache.getQueryData(q2)).to.equal(null); - expect(cache.count()).to.equal(0); + expect(await cache.getQueryCount()).to.equal(0); }); it('can set query to new value', async () => { @@ -279,36 +279,39 @@ function genericQueryCacheTests( ]); }); - it('can get / set highestTargetId', async () => { - expect(cache.getHighestTargetId()).to.deep.equal(0); - const queryData1 = testQueryData(QUERY_ROOMS, 1); + it('can allocate target ID', async () => { + expect(await cache.allocateTargetId()).to.deep.equal(2); + const queryData1 = testQueryData(QUERY_ROOMS, 2); await cache.addQueryData(queryData1); const key1 = key('rooms/bar'); const key2 = key('rooms/foo'); - await cache.addMatchingKeys([key1, key2], 1); + await cache.addMatchingKeys([key1, key2], 2); + + expect(await cache.allocateTargetId()).to.deep.equal(4); - const queryData2 = testQueryData(QUERY_HALLS, 2); + const queryData2 = testQueryData(QUERY_HALLS, 4); await cache.addQueryData(queryData2); const key3 = key('halls/foo'); - await cache.addMatchingKeys([key3], 2); - expect(cache.getHighestTargetId()).to.deep.equal(2); + await cache.addMatchingKeys([key3], 4); + + expect(await cache.allocateTargetId()).to.deep.equal(6); await cache.removeQueryData(queryData2); // Target IDs never come down. - expect(cache.getHighestTargetId()).to.deep.equal(2); + expect(await cache.allocateTargetId()).to.deep.equal(8); // A query with an empty result set still counts. const queryData3 = testQueryData(QUERY_GARAGES, 42); await cache.addQueryData(queryData3); - expect(cache.getHighestTargetId()).to.deep.equal(42); + expect(await cache.allocateTargetId()).to.deep.equal(44); await cache.removeQueryData(queryData1); - expect(cache.getHighestTargetId()).to.deep.equal(42); + expect(await cache.allocateTargetId()).to.deep.equal(46); await cache.removeQueryData(queryData3); - expect(cache.getHighestTargetId()).to.deep.equal(42); + expect(await cache.allocateTargetId()).to.deep.equal(48); // Verify that the highestTargetId persists restarts. const otherCache = new TestQueryCache( @@ -316,19 +319,21 @@ function genericQueryCacheTests( persistence.getQueryCache() ); await otherCache.start(); - expect(otherCache.getHighestTargetId()).to.deep.equal(42); + expect(await otherCache.allocateTargetId()).to.deep.equal(50); }); - it('can get / set lastRemoteSnapshotVersion', () => { - expect(cache.getLastRemoteSnapshotVersion()).to.deep.equal( + it('can get / set targets metadata', async () => { + expect(await cache.getLastRemoteSnapshotVersion()).to.deep.equal( SnapshotVersion.MIN ); // Can set the snapshot version. return cache - .setLastRemoteSnapshotVersion(version(42)) - .then(() => { - expect(cache.getLastRemoteSnapshotVersion()).to.deep.equal(version(42)); + .setTargetsMetadata(/* highestListenSequenceNumber= */ 0, version(42)) + .then(async () => { + expect(await cache.getLastRemoteSnapshotVersion()).to.deep.equal( + version(42) + ); }) .then(() => { // Verify snapshot version persists restarts. @@ -336,8 +341,8 @@ function genericQueryCacheTests( persistence, persistence.getQueryCache() ); - return otherCache.start().then(() => { - expect(otherCache.getLastRemoteSnapshotVersion()).to.deep.equal( + return otherCache.start().then(async () => { + expect(await otherCache.getLastRemoteSnapshotVersion()).to.deep.equal( version(42) ); }); diff --git a/packages/firestore/test/unit/local/remote_document_cache.test.ts b/packages/firestore/test/unit/local/remote_document_cache.test.ts index 6293db77677..bc19e796233 100644 --- a/packages/firestore/test/unit/local/remote_document_cache.test.ts +++ b/packages/firestore/test/unit/local/remote_document_cache.test.ts @@ -19,10 +19,18 @@ import { Query } from '../../../src/core/query'; import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; import { Persistence } from '../../../src/local/persistence'; import { MaybeDocument } from '../../../src/model/document'; -import { deletedDoc, doc, expectEqual, key, path } from '../../util/helpers'; +import { + deletedDoc, + doc, + expectEqual, + key, + path, + removedDoc +} from '../../util/helpers'; import * as persistenceHelpers from './persistence_test_helpers'; import { TestRemoteDocumentCache } from './test_remote_document_cache'; +import { MaybeDocumentMap } from '../../../src/model/collections'; describe('MemoryRemoteDocumentCache', () => { genericRemoteDocumentCacheTests(persistenceHelpers.testMemoryPersistence); @@ -34,7 +42,9 @@ describe('IndexedDbRemoteDocumentCache', () => { return; } - genericRemoteDocumentCacheTests(persistenceHelpers.testIndexedDbPersistence); + genericRemoteDocumentCacheTests(() => + persistenceHelpers.testIndexedDbPersistence(/* synchronizeTabs= */ true) + ); }); /** @@ -55,7 +65,7 @@ function genericRemoteDocumentCacheTests( function setAndReadDocument(doc: MaybeDocument): Promise { return cache - .addEntry(doc) + .addEntries([doc]) .then(() => { return cache.getEntry(doc.key); }) @@ -64,6 +74,24 @@ function genericRemoteDocumentCacheTests( }); } + function assertMatches( + expected: MaybeDocument[], + actual: MaybeDocumentMap + ): void { + expect(actual.size).to.equal(expected.length); + actual.forEach((actualKey, actualDoc) => { + const found = expected.find(expectedDoc => { + if (actualKey.isEqual(expectedDoc.key)) { + expectEqual(actualDoc, expectedDoc); + return true; + } + return false; + }); + + expect(found).to.not.be.undefined; + }); + } + beforeEach(async () => { persistence = await persistencePromise(); cache = new TestRemoteDocumentCache( @@ -93,14 +121,14 @@ function genericRemoteDocumentCacheTests( }); it('can set document to new value', () => { - return cache.addEntry(doc(DOC_PATH, VERSION, DOC_DATA)).then(() => { + return cache.addEntries([doc(DOC_PATH, VERSION, DOC_DATA)]).then(() => { return setAndReadDocument(doc(DOC_PATH, VERSION + 1, { data: 2 })); }); }); it('can remove document', () => { return cache - .addEntry(doc(DOC_PATH, VERSION, DOC_DATA)) + .addEntries([doc(DOC_PATH, VERSION, DOC_DATA)]) .then(() => { return cache.removeEntry(key(DOC_PATH)); }) @@ -117,26 +145,74 @@ function genericRemoteDocumentCacheTests( return cache.removeEntry(key(DOC_PATH)); }); - it('can get documents matching query', () => { + it('can get documents matching query', async () => { // TODO(mikelehen): This just verifies that we do a prefix scan against the // query path. We'll need more tests once we add index support. - return cache - .addEntry(doc('a/1', VERSION, DOC_DATA)) - .then(() => cache.addEntry(doc('b/1', VERSION, DOC_DATA))) - .then(() => cache.addEntry(doc('b/2', VERSION, DOC_DATA))) - .then(() => cache.addEntry(doc('c/1', VERSION, DOC_DATA))) - .then(() => { - const query = new Query(path('b')); - return cache.getDocumentsMatchingQuery(query).then(results => { - const expected = [ - doc('b/1', VERSION, DOC_DATA), - doc('b/2', VERSION, DOC_DATA) - ]; - expect(results.size).to.equal(expected.length); - results.forEach((key, doc) => { - expectEqual(doc, expected.shift()); - }); - }); - }); + await cache.addEntries([ + doc('a/1', VERSION, DOC_DATA), + doc('b/1', VERSION, DOC_DATA), + doc('b/2', VERSION, DOC_DATA), + doc('c/1', VERSION, DOC_DATA) + ]); + + const query = new Query(path('b')); + const matchingDocs = await cache.getDocumentsMatchingQuery(query); + + assertMatches( + [doc('b/1', VERSION, DOC_DATA), doc('b/2', VERSION, DOC_DATA)], + matchingDocs + ); + }); + + it('can get changes', async () => { + await cache.addEntries([ + doc('a/1', 1, DOC_DATA), + doc('b/1', 2, DOC_DATA), + doc('b/2', 2, DOC_DATA), + doc('a/1', 3, DOC_DATA) + ]); + + let changedDocs = await cache.getNextDocumentChanges(); + assertMatches( + [ + doc('a/1', 3, DOC_DATA), + doc('b/1', 2, DOC_DATA), + doc('b/2', 2, DOC_DATA) + ], + changedDocs + ); + + await cache.addEntries([doc('c/1', 3, DOC_DATA)]); + changedDocs = await cache.getNextDocumentChanges(); + assertMatches([doc('c/1', 3, DOC_DATA)], changedDocs); + }); + + it('can get empty changes', async () => { + const changedDocs = await cache.getNextDocumentChanges(); + assertMatches([], changedDocs); + }); + + it('can get missing documents in changes', async () => { + await cache.addEntries([ + doc('a/1', 1, DOC_DATA), + doc('a/2', 2, DOC_DATA), + doc('a/3', 3, DOC_DATA) + ]); + await cache.removeEntry(key('a/2')); + + const changedDocs = await cache.getNextDocumentChanges(); + assertMatches( + [doc('a/1', 1, DOC_DATA), removedDoc('a/2'), doc('a/3', 3, DOC_DATA)], + changedDocs + ); + }); + + it('start() skips previous changes', async () => { + await cache.addEntries([doc('a/1', 1, DOC_DATA)]); + + await cache.start(); + + const changedDocs = await cache.getNextDocumentChanges(); + assertMatches([], changedDocs); }); } diff --git a/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts b/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts index 8105005ab26..ad95e86620c 100644 --- a/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts +++ b/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts @@ -46,10 +46,7 @@ describe('RemoteDocumentChangeBuffer', () => { ); // Add a couple initial items to the cache. - return Promise.all([ - cache.addEntry(INITIAL_DOC), - cache.addEntry(deletedDoc('coll/b', 314)) - ]); + return cache.addEntries([INITIAL_DOC, deletedDoc('coll/b', 314)]); }); }); diff --git a/packages/firestore/test/unit/local/test_garbage_collector.ts b/packages/firestore/test/unit/local/test_garbage_collector.ts index 71bae6e4598..715c4c37d3d 100644 --- a/packages/firestore/test/unit/local/test_garbage_collector.ts +++ b/packages/firestore/test/unit/local/test_garbage_collector.ts @@ -27,7 +27,7 @@ export class TestGarbageCollector { collectGarbage(): Promise { return this.persistence - .runTransaction('garbageCollect', txn => { + .runTransaction('garbageCollect', true, txn => { return this.gc.collectGarbage(txn); }) .then(garbage => { diff --git a/packages/firestore/test/unit/local/test_mutation_queue.ts b/packages/firestore/test/unit/local/test_mutation_queue.ts index d256d6d9416..bfde76d3c8b 100644 --- a/packages/firestore/test/unit/local/test_mutation_queue.ts +++ b/packages/firestore/test/unit/local/test_mutation_queue.ts @@ -34,34 +34,29 @@ export class TestMutationQueue { constructor(public persistence: Persistence, public queue: MutationQueue) {} start(): Promise { - return this.persistence.runTransaction('start', txn => { + return this.persistence.runTransaction('start', true, txn => { return this.queue.start(txn); }); } checkEmpty(): Promise { - return this.persistence.runTransaction('checkEmpty', txn => { + return this.persistence.runTransaction('checkEmpty', true, txn => { return this.queue.checkEmpty(txn); }); } countBatches(): Promise { return this.persistence - .runTransaction('countBatches', txn => { + .runTransaction('countBatches', true, txn => { return this.queue.getAllMutationBatches(txn); }) .then(batches => batches.length); } - getNextBatchId(): Promise { - return this.persistence.runTransaction('getNextBatchId', txn => { - return this.queue.getNextBatchId(txn); - }); - } - getHighestAcknowledgedBatchId(): Promise { return this.persistence.runTransaction( 'getHighestAcknowledgedBatchId', + true, txn => { return this.queue.getHighestAcknowledgedBatchId(txn); } @@ -72,31 +67,35 @@ export class TestMutationQueue { batch: MutationBatch, streamToken: ProtoByteString ): Promise { - return this.persistence.runTransaction('acknowledgeThroughBatchId', txn => { - return this.queue.acknowledgeBatch(txn, batch, streamToken); - }); + return this.persistence.runTransaction( + 'acknowledgeThroughBatchId', + true, + txn => { + return this.queue.acknowledgeBatch(txn, batch, streamToken); + } + ); } getLastStreamToken(): Promise { - return this.persistence.runTransaction('getLastStreamToken', txn => { + return this.persistence.runTransaction('getLastStreamToken', true, txn => { return this.queue.getLastStreamToken(txn); }) as AnyDuringMigration; } setLastStreamToken(streamToken: string): Promise { - return this.persistence.runTransaction('setLastStreamToken', txn => { + return this.persistence.runTransaction('setLastStreamToken', true, txn => { return this.queue.setLastStreamToken(txn, streamToken); }); } addMutationBatch(mutations: Mutation[]): Promise { - return this.persistence.runTransaction('addMutationBatch', txn => { + return this.persistence.runTransaction('addMutationBatch', true, txn => { return this.queue.addMutationBatch(txn, Timestamp.now(), mutations); }); } lookupMutationBatch(batchId: BatchId): Promise { - return this.persistence.runTransaction('lookupMutationBatch', txn => { + return this.persistence.runTransaction('lookupMutationBatch', true, txn => { return this.queue.lookupMutationBatch(txn, batchId); }); } @@ -106,6 +105,7 @@ export class TestMutationQueue { ): Promise { return this.persistence.runTransaction( 'getNextMutationBatchAfterBatchId', + true, txn => { return this.queue.getNextMutationBatchAfterBatchId(txn, batchId); } @@ -113,9 +113,13 @@ export class TestMutationQueue { } getAllMutationBatches(): Promise { - return this.persistence.runTransaction('getAllMutationBatches', txn => { - return this.queue.getAllMutationBatches(txn); - }); + return this.persistence.runTransaction( + 'getAllMutationBatches', + true, + txn => { + return this.queue.getAllMutationBatches(txn); + } + ); } getAllMutationBatchesThroughBatchId( @@ -123,6 +127,7 @@ export class TestMutationQueue { ): Promise { return this.persistence.runTransaction( 'getAllMutationBatchesThroughBatchId', + true, txn => { return this.queue.getAllMutationBatchesThroughBatchId(txn, batchId); } @@ -134,6 +139,7 @@ export class TestMutationQueue { ): Promise { return this.persistence.runTransaction( 'getAllMutationBatchesAffectingDocumentKey', + true, txn => { return this.queue.getAllMutationBatchesAffectingDocumentKey( txn, @@ -148,6 +154,7 @@ export class TestMutationQueue { ): Promise { return this.persistence.runTransaction( 'getAllMutationBatchesAffectingDocumentKeys', + true, txn => { return this.queue.getAllMutationBatchesAffectingDocumentKeys( txn, @@ -160,6 +167,7 @@ export class TestMutationQueue { getAllMutationBatchesAffectingQuery(query: Query): Promise { return this.persistence.runTransaction( 'getAllMutationBatchesAffectingQuery', + true, txn => { return this.queue.getAllMutationBatchesAffectingQuery(txn, query); } @@ -167,13 +175,17 @@ export class TestMutationQueue { } removeMutationBatches(batches: MutationBatch[]): Promise { - return this.persistence.runTransaction('removeMutationBatches', txn => { - return this.queue.removeMutationBatches(txn, batches); - }); + return this.persistence.runTransaction( + 'removeMutationBatches', + true, + txn => { + return this.queue.removeMutationBatches(txn, batches); + } + ); } collectGarbage(gc: GarbageCollector): Promise { - return this.persistence.runTransaction('garbageCollection', txn => { + return this.persistence.runTransaction('garbageCollection', true, txn => { return gc.collectGarbage(txn); }); } diff --git a/packages/firestore/test/unit/local/test_query_cache.ts b/packages/firestore/test/unit/local/test_query_cache.ts index 1023f9967b0..9e915231d48 100644 --- a/packages/firestore/test/unit/local/test_query_cache.ts +++ b/packages/firestore/test/unit/local/test_query_cache.ts @@ -27,53 +27,65 @@ import { DocumentKey } from '../../../src/model/document_key'; * A wrapper around a QueryCache that automatically creates a * transaction around every operation to reduce test boilerplate. */ +// TODO(multitab): Adjust the `requirePrimaryLease` argument to match the usage +// in the client. export class TestQueryCache { constructor(public persistence: Persistence, public cache: QueryCache) {} start(): Promise { - return this.persistence.runTransaction('start', txn => + return this.persistence.runTransaction('start', true, txn => this.cache.start(txn) ); } addQueryData(queryData: QueryData): Promise { - return this.persistence.runTransaction('addQueryData', txn => { + return this.persistence.runTransaction('addQueryData', true, txn => { return this.cache.addQueryData(txn, queryData); }); } updateQueryData(queryData: QueryData): Promise { - return this.persistence.runTransaction('updateQueryData', txn => { + return this.persistence.runTransaction('updateQueryData', true, txn => { return this.cache.updateQueryData(txn, queryData); }); } - count(): number { - return this.cache.count; + getQueryCount(): Promise { + return this.persistence.runTransaction('getQueryCount', true, txn => { + return this.cache.getQueryCount(txn); + }); } removeQueryData(queryData: QueryData): Promise { - return this.persistence.runTransaction('addQueryData', txn => { + return this.persistence.runTransaction('addQueryData', true, txn => { return this.cache.removeQueryData(txn, queryData); }); } getQueryData(query: Query): Promise { - return this.persistence.runTransaction('getQueryData', txn => { + return this.persistence.runTransaction('getQueryData', true, txn => { return this.cache.getQueryData(txn, query); }); } - getLastRemoteSnapshotVersion(): SnapshotVersion { - return this.cache.getLastRemoteSnapshotVersion(); + getLastRemoteSnapshotVersion(): Promise { + return this.persistence.runTransaction( + 'getLastRemoteSnapshotVersion', + true, + txn => { + return this.cache.getLastRemoteSnapshotVersion(txn); + } + ); } - getHighestTargetId(): TargetId { - return this.cache.getHighestTargetId(); + allocateTargetId(): Promise { + return this.persistence.runTransaction('allocateTargetId', false, txn => { + return this.cache.allocateTargetId(txn); + }); } addMatchingKeys(keys: DocumentKey[], targetId: TargetId): Promise { - return this.persistence.runTransaction('addMatchingKeys', txn => { + return this.persistence.runTransaction('addMatchingKeys', true, txn => { let set = documentKeySet(); for (const key of keys) { set = set.add(key); @@ -83,7 +95,7 @@ export class TestQueryCache { } removeMatchingKeys(keys: DocumentKey[], targetId: TargetId): Promise { - return this.persistence.runTransaction('removeMatchingKeys', txn => { + return this.persistence.runTransaction('removeMatchingKeys', true, txn => { let set = documentKeySet(); for (const key of keys) { set = set.add(key); @@ -94,7 +106,7 @@ export class TestQueryCache { getMatchingKeysForTargetId(targetId: TargetId): Promise { return this.persistence - .runTransaction('getMatchingKeysForTargetId', txn => { + .runTransaction('getMatchingKeysForTargetId', true, txn => { return this.cache.getMatchingKeysForTargetId(txn, targetId); }) .then(keySet => { @@ -107,6 +119,7 @@ export class TestQueryCache { removeMatchingKeysForTargetId(targetId: TargetId): Promise { return this.persistence.runTransaction( 'removeMatchingKeysForTargetId', + true, txn => { return this.cache.removeMatchingKeysForTargetId(txn, targetId); } @@ -114,15 +127,21 @@ export class TestQueryCache { } containsKey(key: DocumentKey): Promise { - return this.persistence.runTransaction('containsKey', txn => { + return this.persistence.runTransaction('containsKey', true, txn => { return this.cache.containsKey(txn, key); }); } - setLastRemoteSnapshotVersion(version: SnapshotVersion): Promise { - return this.persistence.runTransaction( - 'setLastRemoteSnapshotVersion', - txn => this.cache.setLastRemoteSnapshotVersion(txn, version) + setTargetsMetadata( + highestListenSequenceNumber: number, + lastRemoteSnapshotVersion?: SnapshotVersion + ): Promise { + return this.persistence.runTransaction('setTargetsMetadata', true, txn => + this.cache.setTargetsMetadata( + txn, + highestListenSequenceNumber, + lastRemoteSnapshotVersion + ) ); } } diff --git a/packages/firestore/test/unit/local/test_remote_document_cache.ts b/packages/firestore/test/unit/local/test_remote_document_cache.ts index 4183c77c30a..30683ecf6e4 100644 --- a/packages/firestore/test/unit/local/test_remote_document_cache.ts +++ b/packages/firestore/test/unit/local/test_remote_document_cache.ts @@ -17,7 +17,7 @@ import { Query } from '../../../src/core/query'; import { Persistence } from '../../../src/local/persistence'; import { RemoteDocumentCache } from '../../../src/local/remote_document_cache'; -import { DocumentMap } from '../../../src/model/collections'; +import { DocumentMap, MaybeDocumentMap } from '../../../src/model/collections'; import { MaybeDocument } from '../../../src/model/document'; import { DocumentKey } from '../../../src/model/document_key'; @@ -31,27 +31,47 @@ export class TestRemoteDocumentCache { public cache: RemoteDocumentCache ) {} - addEntry(maybeDocument: MaybeDocument): Promise { - return this.persistence.runTransaction('addEntry', txn => { - return this.cache.addEntry(txn, maybeDocument); + start(): Promise { + return this.persistence.runTransaction('start', true, txn => { + return this.cache.start(txn); + }); + } + + addEntries(maybeDocuments: MaybeDocument[]): Promise { + return this.persistence.runTransaction('addEntry', true, txn => { + return this.cache.addEntries(txn, maybeDocuments); }); } removeEntry(documentKey: DocumentKey): Promise { - return this.persistence.runTransaction('removeEntry', txn => { + return this.persistence.runTransaction('removeEntry', true, txn => { return this.cache.removeEntry(txn, documentKey); }); } getEntry(documentKey: DocumentKey): Promise { - return this.persistence.runTransaction('getEntry', txn => { + return this.persistence.runTransaction('getEntry', true, txn => { return this.cache.getEntry(txn, documentKey); }); } getDocumentsMatchingQuery(query: Query): Promise { - return this.persistence.runTransaction('getDocumentsMatchingQuery', txn => { - return this.cache.getDocumentsMatchingQuery(txn, query); - }); + return this.persistence.runTransaction( + 'getDocumentsMatchingQuery', + true, + txn => { + return this.cache.getDocumentsMatchingQuery(txn, query); + } + ); + } + + getNextDocumentChanges(): Promise { + return this.persistence.runTransaction( + 'getNextDocumentChanges', + true, + txn => { + return this.cache.getNewDocumentChanges(txn); + } + ); } } diff --git a/packages/firestore/test/unit/local/test_remote_document_change_buffer.ts b/packages/firestore/test/unit/local/test_remote_document_change_buffer.ts index b2ef7918f19..38105d8b275 100644 --- a/packages/firestore/test/unit/local/test_remote_document_change_buffer.ts +++ b/packages/firestore/test/unit/local/test_remote_document_change_buffer.ts @@ -35,13 +35,13 @@ export class TestRemoteDocumentChangeBuffer { } getEntry(documentKey: DocumentKey): Promise { - return this.persistence.runTransaction('getEntry', txn => { + return this.persistence.runTransaction('getEntry', true, txn => { return this.buffer.getEntry(txn, documentKey); }); } apply(): Promise { - return this.persistence.runTransaction('apply', txn => { + return this.persistence.runTransaction('apply', true, txn => { return this.buffer.apply(txn); }); } diff --git a/packages/firestore/test/unit/local/web_storage_shared_client_state.test.ts b/packages/firestore/test/unit/local/web_storage_shared_client_state.test.ts new file mode 100644 index 00000000000..0ccd075b785 --- /dev/null +++ b/packages/firestore/test/unit/local/web_storage_shared_client_state.test.ts @@ -0,0 +1,897 @@ +/** + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * withOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as persistenceHelpers from './persistence_test_helpers'; +import { + WebStorageSharedClientState, + LocalClientState, + MutationMetadata, + ClientId, + SharedClientState, + QueryTargetMetadata +} from '../../../src/local/shared_client_state'; +import { + BatchId, + MutationBatchState, + OnlineState, + TargetId +} from '../../../src/core/types'; +import { AutoId } from '../../../src/util/misc'; +import { expect } from 'chai'; +import { User } from '../../../src/auth/user'; +import { FirestoreError } from '../../../src/util/error'; +import { + SharedClientStateSyncer, + QueryTargetState +} from '../../../src/local/shared_client_state_syncer'; +import { AsyncQueue } from '../../../src/util/async_queue'; +import { + clearWebStorage, + TEST_PERSISTENCE_PREFIX +} from './persistence_test_helpers'; +import { PlatformSupport } from '../../../src/platform/platform'; +import * as objUtils from '../../../src/util/obj'; +import { targetIdSet } from '../../../src/model/collections'; +import { SortedSet } from '../../../src/util/sorted_set'; + +/** + * The tests assert that the lastUpdateTime of each row in LocalStorage gets + * updated. We allow a 0.1s difference in update time to account for processing + * and locking time in LocalStorage. + */ +const GRACE_INTERVAL_MS = 100; + +const AUTHENTICATED_USER = new User('test'); +const UNAUTHENTICATED_USER = User.UNAUTHENTICATED; +const TEST_ERROR = new FirestoreError('internal', 'Test Error'); + +function mutationKey(user: User, batchId: BatchId): string { + if (user.isAuthenticated()) { + return `firestore_mutations_${ + persistenceHelpers.TEST_PERSISTENCE_PREFIX + }_${batchId}_${user.uid}`; + } else { + return `firestore_mutations_${ + persistenceHelpers.TEST_PERSISTENCE_PREFIX + }_${batchId}`; + } +} + +function targetKey(targetId: TargetId): string { + return `firestore_targets_${ + persistenceHelpers.TEST_PERSISTENCE_PREFIX + }_${targetId}`; +} + +function onlineStateKey(): string { + return 'firestore_online_state'; +} + +interface TestSharedClientState { + mutationCount: number; + mutationState: { + [batchId: number]: { state: MutationBatchState; error?: FirestoreError }; + }; + targetIds: SortedSet; + targetState: { + [targetId: number]: { state: QueryTargetState; error?: FirestoreError }; + }; + onlineState: OnlineState; +} + +/** + * Implementation of `SharedClientStateSyncer` that aggregates its callback + * data and exposes it via `.sharedClientState`. + */ +class TestSharedClientSyncer implements SharedClientStateSyncer { + private mutationState: { + [batchId: number]: { state: MutationBatchState; error?: FirestoreError }; + } = {}; + private queryState: { + [targetId: number]: { state: QueryTargetState; error?: FirestoreError }; + } = {}; + private activeTargets = targetIdSet(); + private onlineState = OnlineState.Unknown; + + constructor(public activeClients: ClientId[]) {} + + get sharedClientState(): TestSharedClientState { + return { + mutationCount: objUtils.size(this.mutationState), + mutationState: this.mutationState, + targetIds: this.activeTargets, + targetState: this.queryState, + onlineState: this.onlineState + }; + } + + async applyBatchState( + batchId: BatchId, + state: MutationBatchState, + error?: FirestoreError + ): Promise { + this.mutationState[batchId] = { state, error }; + } + + async applyTargetState( + targetId: TargetId, + state: QueryTargetState, + error?: FirestoreError + ): Promise { + this.queryState[targetId] = { state, error }; + } + + async getActiveClients(): Promise { + return this.activeClients; + } + + async applyActiveTargetsChange( + added: TargetId[], + removed: TargetId[] + ): Promise { + for (const targetId of added) { + expect(this.activeTargets.has(targetId)).to.be.false; + this.activeTargets = this.activeTargets.add(targetId); + } + for (const targetId of removed) { + expect(this.activeTargets.has(targetId)).to.be.true; + this.activeTargets = this.activeTargets.delete(targetId); + } + } + + applyOnlineStateChange(onlineState: OnlineState): void { + this.onlineState = onlineState; + } +} + +describe('WebStorageSharedClientState', () => { + if (!WebStorageSharedClientState.isAvailable(PlatformSupport.getPlatform())) { + console.warn( + 'No LocalStorage. Skipping WebStorageSharedClientState tests.' + ); + return; + } + + const localStorage = window.localStorage; + + let queue: AsyncQueue; + let primaryClientId; + let sharedClientState: SharedClientState; + let clientSyncer: TestSharedClientSyncer; + + let previousAddEventListener; + let previousRemoveEventListener; + + let localStorageCallbacks = []; + + function writeToLocalStorage(key: string, value: string | null): void { + for (const callback of localStorageCallbacks) { + callback({ + key, + storageArea: window.localStorage, + newValue: value + }); + } + } + + beforeEach(() => { + clearWebStorage(); + localStorageCallbacks = []; + + previousAddEventListener = window.addEventListener; + previousRemoveEventListener = window.removeEventListener; + + // We capture the listener here so that we can invoke it from the local + // client. If we directly relied on LocalStorage listeners, we would not + // receive events for local writes. + window.addEventListener = (type, callback) => { + expect(type).to.equal('storage'); + localStorageCallbacks.push(callback); + }; + window.removeEventListener = () => {}; + + primaryClientId = AutoId.newId(); + queue = new AsyncQueue(); + sharedClientState = new WebStorageSharedClientState( + queue, + PlatformSupport.getPlatform(), + TEST_PERSISTENCE_PREFIX, + primaryClientId, + AUTHENTICATED_USER + ); + clientSyncer = new TestSharedClientSyncer([primaryClientId]); + sharedClientState.syncEngine = clientSyncer; + sharedClientState.onlineStateHandler = clientSyncer.applyOnlineStateChange.bind( + clientSyncer + ); + }); + + afterEach(() => { + sharedClientState.shutdown(); + window.addEventListener = previousAddEventListener; + window.removeEventListener = previousRemoveEventListener; + }); + + function assertClientState( + activeTargetIds: number[], + minMutationBatchId: number | null, + maxMutationBatchId: number | null + ): void { + const actual = JSON.parse( + localStorage.getItem( + `firestore_clients_${ + persistenceHelpers.TEST_PERSISTENCE_PREFIX + }_${primaryClientId}` + ) + ); + + expect(Object.keys(actual)).to.have.members([ + 'lastUpdateTime', + 'activeTargetIds', + 'minMutationBatchId', + 'maxMutationBatchId' + ]); + expect(actual.lastUpdateTime) + .to.be.a('number') + .greaterThan(Date.now() - GRACE_INTERVAL_MS) + .and.at.most(Date.now()); + expect(actual.activeTargetIds) + .to.be.an('array') + .and.have.members(activeTargetIds); + expect(actual.minMutationBatchId).to.equal(minMutationBatchId); + expect(actual.maxMutationBatchId).to.equal(maxMutationBatchId); + } + + describe('persists mutation batches', () => { + function assertBatchState( + batchId: BatchId, + mutationBatchState: string, + err?: FirestoreError + ): void { + const actual = JSON.parse( + localStorage.getItem(mutationKey(AUTHENTICATED_USER, batchId)) + ); + + expect(actual.state).to.equal(mutationBatchState); + + const expectedMembers = ['state']; + + if (mutationBatchState === 'rejected') { + expectedMembers.push('error'); + expect(actual.error.code).to.equal(err.code); + expect(actual.error.message).to.equal(err.message); + } + + expect(Object.keys(actual)).to.have.members(expectedMembers); + } + + beforeEach(() => { + return sharedClientState.start(); + }); + + it('when empty', () => { + assertClientState([], null, null); + }); + + it('with one pending batch', () => { + expect(sharedClientState.hasLocalPendingMutation(0)).to.be.false; + assertClientState([], null, null); + + sharedClientState.addLocalPendingMutation(0); + expect(sharedClientState.hasLocalPendingMutation(0)).to.be.true; + assertClientState([], 0, 0); + assertBatchState(0, 'pending'); + + sharedClientState.removeLocalPendingMutation(0); + expect(sharedClientState.hasLocalPendingMutation(0)).to.be.false; + assertClientState([], null, null); + }); + + it('with multiple pending batches', () => { + sharedClientState.addLocalPendingMutation(0); + sharedClientState.addLocalPendingMutation(1); + assertClientState([], 0, 1); + assertBatchState(0, 'pending'); + assertBatchState(1, 'pending'); + + sharedClientState.addLocalPendingMutation(2); + sharedClientState.addLocalPendingMutation(3); + assertClientState([], 0, 3); + assertBatchState(2, 'pending'); + assertBatchState(3, 'pending'); + + // Note: The Firestore client only ever removes mutations in order. + sharedClientState.removeLocalPendingMutation(0); + sharedClientState.removeLocalPendingMutation(2); + assertClientState([], 1, 3); + }); + + it('with an acknowledged batch', () => { + sharedClientState.addLocalPendingMutation(0); + assertClientState([], 0, 0); + assertBatchState(0, 'pending'); + sharedClientState.trackMutationResult(0, 'acknowledged'); + assertBatchState(0, 'acknowledged'); + }); + + it('with a rejected batch', () => { + sharedClientState.addLocalPendingMutation(0); + assertClientState([], 0, 0); + assertBatchState(0, 'pending'); + sharedClientState.trackMutationResult(0, 'rejected', TEST_ERROR); + assertBatchState(0, 'rejected', TEST_ERROR); + }); + }); + + describe('persists query targets', () => { + function assertTargetState( + targetId: TargetId, + queryTargetState: string, + err?: FirestoreError + ): void { + if (queryTargetState === 'pending') { + expect(localStorage.getItem(targetKey(targetId))).to.be.null; + } else { + const actual = JSON.parse(localStorage.getItem(targetKey(targetId))); + expect(actual.state).to.equal(queryTargetState); + + const expectedMembers = ['state', 'lastUpdateTime']; + if (queryTargetState === 'rejected') { + expectedMembers.push('error'); + expect(actual.error.code).to.equal(err.code); + expect(actual.error.message).to.equal(err.message); + } + expect(Object.keys(actual)).to.have.members(expectedMembers); + } + } + + beforeEach(() => { + return sharedClientState.start(); + }); + + it('when empty', () => { + assertClientState([], null, null); + }); + + it('with multiple targets', () => { + sharedClientState.addLocalQueryTarget(0); + assertClientState([0], null, null); + assertTargetState(0, 'pending'); + + sharedClientState.addLocalQueryTarget(1); + sharedClientState.addLocalQueryTarget(2); + assertClientState([0, 1, 2], null, null); + assertTargetState(1, 'pending'); + assertTargetState(2, 'pending'); + + sharedClientState.removeLocalQueryTarget(1); + assertClientState([0, 2], null, null); + }); + + it('with a not-current target', () => { + sharedClientState.addLocalQueryTarget(0); + assertClientState([0], null, null); + assertTargetState(0, 'pending'); + sharedClientState.trackQueryUpdate(0, 'not-current'); + assertTargetState(0, 'not-current'); + }); + + it('with a current target', () => { + sharedClientState.addLocalQueryTarget(0); + assertClientState([0], null, null); + assertTargetState(0, 'pending'); + sharedClientState.trackQueryUpdate(0, 'not-current'); + assertTargetState(0, 'not-current'); + sharedClientState.trackQueryUpdate(0, 'current'); + assertTargetState(0, 'current'); + }); + + it('with an errored target', () => { + sharedClientState.addLocalQueryTarget(0); + assertClientState([0], null, null); + assertTargetState(0, 'pending'); + sharedClientState.trackQueryUpdate(0, 'rejected', TEST_ERROR); + assertTargetState(0, 'rejected', TEST_ERROR); + }); + }); + + describe('combines client state', () => { + const secondaryClientId = AutoId.newId(); + const secondaryClientStateKey = `firestore_clients_${ + persistenceHelpers.TEST_PERSISTENCE_PREFIX + }_${secondaryClientId}`; + + beforeEach(() => { + const existingClientId = AutoId.newId(); + + return persistenceHelpers + .populateWebStorage( + AUTHENTICATED_USER, + existingClientId, + [1, 2], + [3, 4] + ) + .then(() => { + clientSyncer.activeClients = [primaryClientId, existingClientId]; + return sharedClientState.start(); + }); + }); + + async function verifyState( + minBatchId: BatchId | null, + expectedTargets: TargetId[], + expectedOnlineState: OnlineState + ): Promise { + await queue.drain(); + const actualOnlineState = clientSyncer.sharedClientState.onlineState; + const actualTargets = sharedClientState.getAllActiveQueryTargets(); + + expect(actualTargets.toArray()).to.have.members(expectedTargets); + expect(sharedClientState.getMinimumGlobalPendingMutation()).to.equal( + minBatchId + ); + expect(actualOnlineState).to.equal(expectedOnlineState); + } + + it('with targets and mutations from existing client', async () => { + // The prior client has one pending mutation and two active query targets + await verifyState(1, [3, 4], OnlineState.Unknown); + + sharedClientState.addLocalPendingMutation(3); + sharedClientState.addLocalQueryTarget(4); + await verifyState(1, [3, 4], OnlineState.Unknown); + + // This is technically invalid as IDs of minimum mutation batches should + // never decrease over the lifetime of a client, but we use it here to + // test the underlying logic that extracts the mutation batch IDs. + sharedClientState.addLocalPendingMutation(0); + sharedClientState.addLocalQueryTarget(5); + await verifyState(0, [3, 4, 5], OnlineState.Unknown); + + sharedClientState.removeLocalPendingMutation(0); + sharedClientState.removeLocalQueryTarget(5); + await verifyState(1, [3, 4], OnlineState.Unknown); + }); + + it('with targets from new client', async () => { + // The prior client has one pending mutation and two active query targets + await verifyState(1, [3, 4], OnlineState.Unknown); + + const oldState = new LocalClientState(); + oldState.addQueryTarget(5); + + writeToLocalStorage( + secondaryClientStateKey, + oldState.toLocalStorageJSON() + ); + await verifyState(1, [3, 4, 5], OnlineState.Unknown); + + const updatedState = new LocalClientState(); + updatedState.addQueryTarget(5); + updatedState.addQueryTarget(6); + + writeToLocalStorage( + secondaryClientStateKey, + updatedState.toLocalStorageJSON() + ); + await verifyState(1, [3, 4, 5, 6], OnlineState.Unknown); + + writeToLocalStorage(secondaryClientStateKey, null); + await verifyState(1, [3, 4], OnlineState.Unknown); + }); + + it('with mutations from new client', async () => { + // The prior client has one pending mutation and two active query targets + await verifyState(1, [3, 4], OnlineState.Unknown); + + const updatedState = new LocalClientState(); + updatedState.addPendingMutation(0); + + writeToLocalStorage( + secondaryClientStateKey, + updatedState.toLocalStorageJSON() + ); + await verifyState(0, [3, 4], OnlineState.Unknown); + + writeToLocalStorage(secondaryClientStateKey, null); + await verifyState(1, [3, 4], OnlineState.Unknown); + }); + + it('with online state from new client', async () => { + // The prior client has one pending mutation and two active query targets + await verifyState(1, [3, 4], OnlineState.Unknown); + + // Ensure that client is considered active + const oldState = new LocalClientState(); + writeToLocalStorage( + secondaryClientStateKey, + oldState.toLocalStorageJSON() + ); + + writeToLocalStorage( + onlineStateKey(), + JSON.stringify({ + onlineState: 'Unknown', + clientId: secondaryClientId + }) + ); + await verifyState(1, [3, 4], OnlineState.Unknown); + + writeToLocalStorage( + onlineStateKey(), + JSON.stringify({ + onlineState: 'Offline', + clientId: secondaryClientId + }) + ); + await verifyState(1, [3, 4], OnlineState.Offline); + + writeToLocalStorage( + onlineStateKey(), + JSON.stringify({ + onlineState: 'Online', + clientId: secondaryClientId + }) + ); + await verifyState(1, [3, 4], OnlineState.Online); + }); + + it('ignores online state from inactive client', async () => { + // The prior client has one pending mutation and two active query targets + await verifyState(1, [3, 4], OnlineState.Unknown); + + // The secondary client is inactive and its online state is ignored. + writeToLocalStorage( + onlineStateKey(), + JSON.stringify({ + onlineState: 'Online', + clientId: secondaryClientId + }) + ); + + await verifyState(1, [3, 4], OnlineState.Unknown); + + // Ensure that client is considered active + const oldState = new LocalClientState(); + writeToLocalStorage( + secondaryClientStateKey, + oldState.toLocalStorageJSON() + ); + + writeToLocalStorage( + onlineStateKey(), + JSON.stringify({ + onlineState: 'Online', + clientId: secondaryClientId + }) + ); + + await verifyState(1, [3, 4], OnlineState.Online); + }); + + it('ignores invalid data', async () => { + const secondaryClientStateKey = `firestore_clients_${ + persistenceHelpers.TEST_PERSISTENCE_PREFIX + }_${AutoId.newId()}`; + + const invalidState = { + lastUpdateTime: 'invalid', + activeTargetIds: [5] + }; + + // The prior instance has one pending mutation and two active query targets + await verifyState(1, [3, 4], OnlineState.Unknown); + + // We ignore the newly added target. + writeToLocalStorage( + secondaryClientStateKey, + JSON.stringify(invalidState) + ); + await verifyState(1, [3, 4], OnlineState.Unknown); + }); + }); + + describe('processes mutation updates', () => { + beforeEach(() => { + return sharedClientState.start(); + }); + + async function withUser( + user: User, + fn: () => Promise + ): Promise { + await sharedClientState.handleUserChange(user, [], []); + await fn(); + await queue.drain(); + return clientSyncer.sharedClientState; + } + + it('for pending mutation', () => { + return withUser(AUTHENTICATED_USER, async () => { + writeToLocalStorage( + mutationKey(AUTHENTICATED_USER, 1), + new MutationMetadata( + AUTHENTICATED_USER, + 1, + 'pending' + ).toLocalStorageJSON() + ); + }).then(clientState => { + expect(clientState.mutationCount).to.equal(1); + expect(clientState.mutationState[1].state).to.equal('pending'); + }); + }); + + it('for acknowledged mutation', () => { + return withUser(AUTHENTICATED_USER, async () => { + writeToLocalStorage( + mutationKey(AUTHENTICATED_USER, 1), + new MutationMetadata( + AUTHENTICATED_USER, + 1, + 'acknowledged' + ).toLocalStorageJSON() + ); + }).then(clientState => { + expect(clientState.mutationCount).to.equal(1); + expect(clientState.mutationState[1].state).to.equal('acknowledged'); + }); + }); + + it('for rejected mutation', () => { + return withUser(AUTHENTICATED_USER, async () => { + writeToLocalStorage( + mutationKey(AUTHENTICATED_USER, 1), + new MutationMetadata( + AUTHENTICATED_USER, + 1, + 'rejected', + TEST_ERROR + ).toLocalStorageJSON() + ); + }).then(clientState => { + expect(clientState.mutationCount).to.equal(1); + expect(clientState.mutationState[1].state).to.equal('rejected'); + + const firestoreError = clientState.mutationState[1].error; + expect(firestoreError.code).to.equal('internal'); + expect(firestoreError.message).to.equal('Test Error'); + }); + }); + + it('handles unauthenticated user', () => { + return withUser(UNAUTHENTICATED_USER, async () => { + writeToLocalStorage( + mutationKey(UNAUTHENTICATED_USER, 1), + new MutationMetadata( + UNAUTHENTICATED_USER, + 1, + 'pending' + ).toLocalStorageJSON() + ); + }).then(clientState => { + expect(clientState.mutationCount).to.equal(1); + expect(clientState.mutationState[1].state).to.equal('pending'); + }); + }); + + it('ignores different user', () => { + return withUser(AUTHENTICATED_USER, async () => { + const otherUser = new User('foobar'); + + writeToLocalStorage( + mutationKey(AUTHENTICATED_USER, 1), + new MutationMetadata( + AUTHENTICATED_USER, + 1, + 'pending' + ).toLocalStorageJSON() + ); + writeToLocalStorage( + mutationKey(otherUser, 1), + new MutationMetadata(otherUser, 2, 'pending').toLocalStorageJSON() + ); + }).then(clientState => { + expect(clientState.mutationCount).to.equal(1); + expect(clientState.mutationState[1].state).to.equal('pending'); + }); + }); + + it('ignores invalid data', () => { + return withUser(AUTHENTICATED_USER, async () => { + writeToLocalStorage( + mutationKey(AUTHENTICATED_USER, 1), + new MutationMetadata( + AUTHENTICATED_USER, + 1, + 'invalid' as any // tslint:disable-line:no-any + ).toLocalStorageJSON() + ); + }).then(clientState => { + expect(clientState.mutationCount).to.equal(0); + }); + }); + }); + + describe('processes target updates', () => { + const firstClientTargetId: TargetId = 1; + const secondClientTargetId: TargetId = 2; + + const firstClientStorageKey = `firestore_clients_${ + persistenceHelpers.TEST_PERSISTENCE_PREFIX + }_${AutoId.newId()}`; + const secondClientStorageKey = `firestore_clients_${ + persistenceHelpers.TEST_PERSISTENCE_PREFIX + }_${AutoId.newId()}`; + + let firstClient: LocalClientState; + let secondClientState: LocalClientState; + + beforeEach(() => { + firstClient = new LocalClientState(); + firstClient.addQueryTarget(firstClientTargetId); + secondClientState = new LocalClientState(); + return sharedClientState.start(); + }); + + async function withClientState( + fn: () => Promise + ): Promise { + writeToLocalStorage( + firstClientStorageKey, + firstClient.toLocalStorageJSON() + ); + await fn(); + await queue.drain(); + return clientSyncer.sharedClientState; + } + + it('for added target', async () => { + let clientState = await withClientState(async () => { + // Add a target that only exists in the second client + secondClientState.addQueryTarget(secondClientTargetId); + writeToLocalStorage( + secondClientStorageKey, + secondClientState.toLocalStorageJSON() + ); + }); + + expect(clientState.targetIds.size).to.equal(2); + + clientState = await withClientState(async () => { + // Add a target that already exist in the first client + secondClientState.addQueryTarget(firstClientTargetId); + writeToLocalStorage( + secondClientStorageKey, + secondClientState.toLocalStorageJSON() + ); + }); + + expect(clientState.targetIds.size).to.equal(2); + }); + + it('for removed target', async () => { + let clientState = await withClientState(async () => { + secondClientState.addQueryTarget(firstClientTargetId); + secondClientState.addQueryTarget(secondClientTargetId); + writeToLocalStorage( + secondClientStorageKey, + secondClientState.toLocalStorageJSON() + ); + }); + + expect(clientState.targetIds.size).to.equal(2); + + clientState = await withClientState(async () => { + // Remove a target that also exists in the first client + secondClientState.removeQueryTarget(firstClientTargetId); + writeToLocalStorage( + secondClientStorageKey, + secondClientState.toLocalStorageJSON() + ); + }); + + expect(clientState.targetIds.size).to.equal(2); + + clientState = await withClientState(async () => { + // Remove a target that only exists in the second client + secondClientState.removeQueryTarget(secondClientTargetId); + writeToLocalStorage( + secondClientStorageKey, + secondClientState.toLocalStorageJSON() + ); + }); + + expect(clientState.targetIds.size).to.equal(1); + }); + + it('for not-current target', () => { + return withClientState(async () => { + writeToLocalStorage( + targetKey(firstClientTargetId), + new QueryTargetMetadata( + firstClientTargetId, + new Date(), + 'not-current' + ).toLocalStorageJSON() + ); + }).then(clientState => { + expect(clientState.targetIds.size).to.equal(1); + expect(clientState.targetState[firstClientTargetId].state).to.equal( + 'not-current' + ); + }); + }); + + it('for current target', () => { + return withClientState(async () => { + writeToLocalStorage( + targetKey(firstClientTargetId), + new QueryTargetMetadata( + firstClientTargetId, + new Date(), + 'current' + ).toLocalStorageJSON() + ); + }).then(clientState => { + expect(clientState.targetIds.size).to.equal(1); + expect(clientState.targetState[firstClientTargetId].state).to.equal( + 'current' + ); + }); + }); + + it('for errored target', () => { + return withClientState(async () => { + writeToLocalStorage( + targetKey(1), + new QueryTargetMetadata( + firstClientTargetId, + new Date(), + 'rejected', + TEST_ERROR + ).toLocalStorageJSON() + ); + }).then(clientState => { + expect(clientState.targetIds.size).to.equal(1); + expect(clientState.targetState[firstClientTargetId].state).to.equal( + 'rejected' + ); + + const firestoreError = + clientState.targetState[firstClientTargetId].error; + expect(firestoreError.code).to.equal('internal'); + expect(firestoreError.message).to.equal('Test Error'); + }); + }); + + it('ignores invalid data', () => { + return withClientState(async () => { + writeToLocalStorage( + targetKey(firstClientTargetId), + new QueryTargetMetadata( + firstClientTargetId, + new Date(), + 'invalid' as any // tslint:disable-line:no-any + ).toLocalStorageJSON() + ); + }).then(clientState => { + expect(clientState.targetIds.size).to.equal(1); + expect(clientState.targetState[firstClientTargetId]).to.be.undefined; + }); + }); + }); +}); diff --git a/packages/firestore/test/unit/specs/describe_spec.ts b/packages/firestore/test/unit/specs/describe_spec.ts index 21db2a6d123..60fdeac7e01 100644 --- a/packages/firestore/test/unit/specs/describe_spec.ts +++ b/packages/firestore/test/unit/specs/describe_spec.ts @@ -24,22 +24,22 @@ import { SpecStep } from './spec_test_runner'; // Disables all other tests; useful for debugging. Multiple tests can have // this tag and they'll all be run (but all others won't). const EXCLUSIVE_TAG = 'exclusive'; -// Persistence-related tests. -const PERSISTENCE_TAG = 'persistence'; +// Multi-client related tests (which imply persistence). +const MULTI_CLIENT_TAG = 'multi-client'; // Explicit per-platform disable flags. const NO_WEB_TAG = 'no-web'; const NO_ANDROID_TAG = 'no-android'; const NO_IOS_TAG = 'no-ios'; -const NO_LRU = 'no-lru'; +const NO_LRU_TAG = 'no-lru'; const BENCHMARK_TAG = 'benchmark'; const KNOWN_TAGS = [ BENCHMARK_TAG, EXCLUSIVE_TAG, - PERSISTENCE_TAG, + MULTI_CLIENT_TAG, NO_WEB_TAG, NO_ANDROID_TAG, NO_IOS_TAG, - NO_LRU + NO_LRU_TAG ]; // TOOD(mrschmidt): Make this configurable with mocha options. @@ -74,6 +74,24 @@ export function setSpecJSONHandler(writer: (json: string) => void): void { writeJSONFile = writer; } +/** Gets the test runner based on the specified tags. */ +function getTestRunner(tags, persistenceEnabled): Function { + if (tags.indexOf(NO_WEB_TAG) >= 0) { + return it.skip; + } else if (persistenceEnabled && tags.indexOf(NO_LRU_TAG) !== -1) { + // spec should have a comment explaining why it is being skipped. + return it.skip; + } else if (!persistenceEnabled && tags.indexOf(MULTI_CLIENT_TAG) !== -1) { + return it.skip; + } else if (tags.indexOf(BENCHMARK_TAG) >= 0 && !RUN_BENCHMARK_TESTS) { + return it.skip; + } else if (tags.indexOf(EXCLUSIVE_TAG) >= 0) { + return it.only; + } else { + return it; + } +} + /** * Like it(), but for spec tests. * @param name A name to give the test. @@ -124,19 +142,7 @@ export function specTest( : [false]; for (const usePersistence of persistenceModes) { const spec = builder(); - let runner: Function; - if (tags.indexOf(EXCLUSIVE_TAG) >= 0) { - runner = it.only; - } else if (tags.indexOf(NO_WEB_TAG) >= 0) { - runner = it.skip; - } else if (tags.indexOf(BENCHMARK_TAG) >= 0 && !RUN_BENCHMARK_TESTS) { - runner = it.skip; - } else if (usePersistence && tags.indexOf('no-lru') !== -1) { - // spec should have a comment explaining why it is being skipped. - runner = it.skip; - } else { - runner = it; - } + const runner = getTestRunner(tags, usePersistence); const mode = usePersistence ? '(Persistence)' : '(Memory)'; const fullName = `${mode} ${name}`; runner(fullName, async () => { diff --git a/packages/firestore/test/unit/specs/limbo_spec.test.ts b/packages/firestore/test/unit/specs/limbo_spec.test.ts index 8e1d98c36d0..6ba4f233137 100644 --- a/packages/firestore/test/unit/specs/limbo_spec.test.ts +++ b/packages/firestore/test/unit/specs/limbo_spec.test.ts @@ -18,7 +18,8 @@ import { Query } from '../../../src/core/query'; import { deletedDoc, doc, filter, path } from '../../util/helpers'; import { describeSpec, specTest } from './describe_spec'; -import { spec } from './spec_builder'; +import { client, spec } from './spec_builder'; +import { TimerId } from '../../../src/util/async_queue'; describeSpec('Limbo Documents:', [], () => { specTest( @@ -319,4 +320,117 @@ describeSpec('Limbo Documents:', [], () => { ); } ); + + specTest( + 'Limbo docs are resolved by primary client', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + const docB = doc('collection/b', 1001, { key: 'b' }); + const deletedDocB = deletedDoc('collection/b', 1004); + + return client(0) + .expectPrimaryState(true) + .client(1) + .userListens(query) + .client(0) + .expectListen(query) + .watchAcksFull(query, 1002, docA, docB) + .client(1) + .expectEvents(query, { added: [docA, docB] }) + .client(0) + .watchRemovesDoc(docB.key, query) + .watchSnapshots(1003) + .expectLimboDocs(docB.key) + .client(1) + .expectEvents(query, { fromCache: true }) + .client(0) + .ackLimbo(1004, deletedDocB) + .expectLimboDocs() + .client(1) + .expectEvents(query, { removed: [docB] }); + } + ); + + specTest( + 'Limbo documents are resolved after primary tab failover', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + const docB = doc('collection/b', 1001, { key: 'b' }); + const deletedDocB = deletedDoc('collection/b', 1005); + + return client(0, false) + .expectPrimaryState(true) + .client(1) + .userListens(query) + .client(0) + .expectListen(query) + .watchAcksFull(query, 1 * 1e6, docA, docB) + .client(1) + .expectEvents(query, { added: [docA, docB] }) + .client(0) + .watchRemovesDoc(docB.key, query) + .watchSnapshots(2 * 1e6) + .expectLimboDocs(docB.key) + .shutdown() + .client(1) + .expectEvents(query, { fromCache: true }) + .runTimer(TimerId.ClientMetadataRefresh) + .expectPrimaryState(true) + .expectListen(query, 'resume-token-1000000') + .watchAcksFull(query, 3 * 1e6) + .expectLimboDocs(docB.key) + .ackLimbo(4 * 1e6, deletedDocB) + .expectLimboDocs() + .expectEvents(query, { removed: [docB] }); + } + ); + + specTest( + 'Limbo documents survive primary state transitions', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + const docB = doc('collection/b', 1001, { key: 'b' }); + const docC = doc('collection/c', 1002, { key: 'c' }); + const deletedDocB = deletedDoc('collection/b', 1006); + const deletedDocC = deletedDoc('collection/c', 1008); + + return client(0, false) + .expectPrimaryState(true) + .userListens(query) + .watchAcksFull(query, 1 * 1e6, docA, docB, docC) + .expectEvents(query, { added: [docA, docB, docC] }) + .watchRemovesDoc(docB.key, query) + .watchRemovesDoc(docC.key, query) + .watchSnapshots(2 * 1e6) + .expectEvents(query, { fromCache: true }) + .expectLimboDocs(docB.key, docC.key) + .client(1) + .stealPrimaryLease() + .client(0) + .runTimer(TimerId.ClientMetadataRefresh) + .expectPrimaryState(false) + .expectLimboDocs() + .client(1) + .expectListen(query, 'resume-token-1000000') + .watchAcksFull(query, 3 * 1e6) + .expectLimboDocs(docB.key, docC.key) + .ackLimbo(3 * 1e6, deletedDocB) + .expectLimboDocs(docC.key) + .client(0) + .expectEvents(query, { removed: [docB], fromCache: true }) + .stealPrimaryLease() + .expectListen(query, 'resume-token-1000000') + .watchAcksFull(query, 5 * 1e6) + .expectLimboDocs(docC.key) + .ackLimbo(6 * 1e6, deletedDocC) + .expectLimboDocs() + .expectEvents(query, { removed: [docC] }); + } + ); }); diff --git a/packages/firestore/test/unit/specs/limit_spec.test.ts b/packages/firestore/test/unit/specs/limit_spec.test.ts index 861015b731e..0668a053148 100644 --- a/packages/firestore/test/unit/specs/limit_spec.test.ts +++ b/packages/firestore/test/unit/specs/limit_spec.test.ts @@ -18,7 +18,7 @@ import { Query } from '../../../src/core/query'; import { deletedDoc, doc, path } from '../../util/helpers'; import { describeSpec, specTest } from './describe_spec'; -import { spec } from './spec_builder'; +import { client, spec } from './spec_builder'; describeSpec('Limits:', [], () => { specTest('Documents in limit are replaced by remote event', [], () => { @@ -217,4 +217,84 @@ describeSpec('Limits:', [], () => { .watchRemovesLimboTarget(docD) ); }); + + specTest( + 'Limit query is refilled by primary client', + ['multi-client'], + () => { + const query1 = Query.atPath(path('collection')).withLimit(2); + const doc1 = doc('collection/a', 1000, { key: 'a' }); + const doc2 = doc('collection/b', 1002, { key: 'b' }); + const doc3 = doc('collection/c', 1001, { key: 'c' }); + return client(0) + .becomeVisible() + .client(1) + .userListens(query1) + .client(0) + .expectListen(query1) + .watchAcksFull(query1, 1001, doc1, doc3) + .client(1) + .expectEvents(query1, { + added: [doc1, doc3] + }) + .client(0) + .watchSends({ affects: [query1] }, doc2) + .watchSends({ removed: [query1] }, doc3) + .watchSnapshots(1002) + .client(1) + .expectEvents(query1, { + added: [doc2], + removed: [doc3] + }); + } + ); + + specTest( + 'Limit query includes write from secondary client ', + ['multi-client'], + () => { + const query1 = Query.atPath(path('collection')).withLimit(2); + const doc1 = doc('collection/a', 1003, { key: 'a' }); + const doc1Local = doc( + 'collection/a', + 0, + { key: 'a' }, + { hasLocalMutations: true } + ); + const doc2 = doc('collection/b', 1001, { key: 'b' }); + const doc3 = doc('collection/c', 1002, { key: 'c' }); + return client(0) + .becomeVisible() + .client(1) + .userListens(query1) + .client(0) + .expectListen(query1) + .watchAcksFull(query1, 1002, doc2, doc3) + .client(1) + .expectEvents(query1, { + added: [doc2, doc3] + }) + .client(2) + .userSets('collection/a', { key: 'a' }) + .client(1) + .expectEvents(query1, { + hasPendingWrites: true, + added: [doc1Local], + removed: [doc3] + }) + .client(0) + .writeAcks('collection/a', 1003, { expectUserCallback: false }) + .watchSends({ affects: [query1] }, doc1) + .watchSends({ removed: [query1] }, doc3) + .watchSnapshots(1003) + .client(1) + .expectEvents(query1, { + metadata: [doc1] + }) + .client(2) + .expectUserCallbacks({ + acknowledged: ['collection/a'] + }); + } + ); }); diff --git a/packages/firestore/test/unit/specs/listen_spec.test.ts b/packages/firestore/test/unit/specs/listen_spec.test.ts index 1442cfedd04..a1864515284 100644 --- a/packages/firestore/test/unit/specs/listen_spec.test.ts +++ b/packages/firestore/test/unit/specs/listen_spec.test.ts @@ -19,8 +19,9 @@ import { Code } from '../../../src/util/error'; import { deletedDoc, doc, filter, path } from '../../util/helpers'; import { describeSpec, specTest } from './describe_spec'; -import { spec } from './spec_builder'; +import { client, spec } from './spec_builder'; import { RpcError } from './spec_rpc_error'; +import { TimerId } from '../../../src/util/async_queue'; describeSpec('Listens:', [], () => { // Obviously this test won't hold with offline persistence enabled. @@ -141,6 +142,23 @@ describeSpec('Listens:', [], () => { } ); + specTest('Will re-issue listen for errored target', [], () => { + const query = Query.atPath(path('collection')); + + return spec() + .withGCEnabled(false) + .userListens(query) + .watchAcks(query) + .watchRemoves( + query, + new RpcError(Code.RESOURCE_EXHAUSTED, 'Resource exhausted') + ) + .expectEvents(query, { errorCode: Code.RESOURCE_EXHAUSTED }) + .userListens(query) + .watchAcksFull(query, 1000) + .expectEvents(query, {}); + }); + // It can happen that we need to process watch messages for previously failed // targets, because target failures are handled out of band. // This test verifies that the code does not crash in this case. @@ -490,6 +508,22 @@ describeSpec('Listens:', [], () => { } ); + specTest('Query is rejected and re-listened to', [], () => { + const query = Query.atPath(path('collection')); + + return spec() + .withGCEnabled(false) + .userListens(query) + .watchRemoves( + query, + new RpcError(Code.RESOURCE_EXHAUSTED, 'Resource exhausted') + ) + .expectEvents(query, { errorCode: Code.RESOURCE_EXHAUSTED }) + .userListens(query) + .watchAcksFull(query, 1000) + .expectEvents(query, {}); + }); + specTest('Persists resume token sent with target', [], () => { const query = Query.atPath(path('collection')); const docA = doc('collection/a', 2000, { key: 'a' }); @@ -593,4 +627,490 @@ describeSpec('Listens:', [], () => { ); } ); + + specTest('Query is executed by primary client', ['multi-client'], () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + + return client(0) + .becomeVisible() + .client(1) + .userListens(query) + .client(0) + .expectListen(query) + .watchAcks(query) + .watchSends({ affects: [query] }, docA) + .watchSnapshots(1000) + .client(1) + .expectEvents(query, { added: [docA], fromCache: true }) + .client(0) + .watchCurrents(query, 'resume-token-2000') + .watchSnapshots(2000) + .client(1) + .expectEvents(query, { fromCache: false }); + }); + + specTest( + 'Query is shared between primary and secondary client', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + const docB = doc('collection/b', 2000, { key: 'a' }); + + return client(0) + .becomeVisible() + .userListens(query) + .watchAcksFull(query, 1000, docA) + .expectEvents(query, { added: [docA] }) + .client(1) + .userListens(query) + .expectEvents(query, { added: [docA] }) + .client(2) + .userListens(query) + .expectEvents(query, { added: [docA] }) + .client(0) + .watchSends({ affects: [query] }, docB) + .watchSnapshots(2000) + .expectEvents(query, { added: [docB] }) + .client(1) + .expectEvents(query, { added: [docB] }) + .client(2) + .expectEvents(query, { added: [docB] }); + } + ); + + specTest('Query is joined by primary client', ['multi-client'], () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + const docB = doc('collection/b', 2000, { key: 'b' }); + const docC = doc('collection/c', 3000, { key: 'c' }); + + return client(0) + .expectPrimaryState(true) + .client(1) + .userListens(query) + .client(0) + .expectListen(query) + .watchAcksFull(query, 100, docA) + .client(1) + .expectEvents(query, { added: [docA] }) + .client(0) + .watchSends({ affects: [query] }, docB) + .watchSnapshots(2000) + .userListens(query) + .expectEvents(query, { added: [docA, docB] }) + .watchSends({ affects: [query] }, docC) + .watchSnapshots(3000) + .expectEvents(query, { added: [docC] }) + .client(1) + .expectEvents(query, { added: [docB] }) + .expectEvents(query, { added: [docC] }); + }); + + specTest( + 'Query only raises events in participating clients', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + + return client(0) + .becomeVisible() + .client(1) + .client(2) + .userListens(query) + .client(3) + .userListens(query) + .client(0) // No events + .expectListen(query) + .watchAcksFull(query, 1000, docA) + .client(1) // No events + .client(2) + .expectEvents(query, { added: [docA] }) + .client(3) + .expectEvents(query, { added: [docA] }); + } + ); + + specTest('Query is unlistened to by primary client', ['multi-client'], () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + const docB = doc('collection/b', 2000, { key: 'a' }); + + return client(0) + .becomeVisible() + .userListens(query) + .watchAcksFull(query, 1000, docA) + .expectEvents(query, { added: [docA] }) + .client(1) + .userListens(query) + .expectEvents(query, { added: [docA] }) + .client(0) + .userUnlistens(query) + .expectListen(query) + .watchSends({ affects: [query] }, docB) + .watchSnapshots(2000) + .client(1) + .expectEvents(query, { added: [docB] }) + .userUnlistens(query) + .client(0) + .expectUnlisten(query); + }); + + specTest('Query is resumed by secondary client', ['multi-client'], () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + const docB = doc('collection/b', 2000, { key: 'a' }); + + return client(0, /* withGcEnabled= */ false) + .becomeVisible() + .client(1) + .userListens(query) + .client(0) + .expectListen(query) + .watchAcksFull(query, 1000, docA) + .client(1) + .expectEvents(query, { added: [docA] }) + .userUnlistens(query) + .client(0) + .expectUnlisten(query) + .watchRemoves(query) + .client(1) + .userListens(query) + .expectEvents(query, { added: [docA], fromCache: true }) + .client(0) + .expectListen(query, 'resume-token-1000') + .watchAcksFull(query, 2000, docB) + .client(1) + .expectEvents(query, { added: [docB] }); + }); + + specTest('Query is rejected by primary client', ['multi-client'], () => { + const query = Query.atPath(path('collection')); + + return client(0) + .becomeVisible() + .client(1) + .userListens(query) + .client(0) + .expectListen(query) + .watchRemoves( + query, + new RpcError(Code.RESOURCE_EXHAUSTED, 'Resource exhausted') + ) + .client(1) + .expectEvents(query, { errorCode: Code.RESOURCE_EXHAUSTED }); + }); + + specTest( + 'Query is rejected and re-listened to by secondary client', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + + return client(0) + .becomeVisible() + .client(1) + .userListens(query) + .client(0) + .expectListen(query) + .watchRemoves( + query, + new RpcError(Code.RESOURCE_EXHAUSTED, 'Resource exhausted') + ) + .client(1) + .expectEvents(query, { errorCode: Code.RESOURCE_EXHAUSTED }) + .userListens(query) + .client(0) + .expectListen(query) + .watchAcksFull(query, 1000) + .client(1) + .expectEvents(query, {}); + } + ); + + specTest( + "Secondary client uses primary client's online state", + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + + return client(0) + .becomeVisible() + .client(1) + .userListens(query) + .client(0) + .expectListen(query) + .watchAcksFull(query, 1000) + .client(1) + .expectEvents(query, {}) + .client(0) + .disableNetwork() + .client(1) + .expectEvents(query, { fromCache: true }) + .client(0) + .enableNetwork() + .expectListen(query, 'resume-token-1000') + .watchAcksFull(query, 2000) + .client(1) + .expectEvents(query, {}); + } + ); + + specTest('New client uses existing online state', ['multi-client'], () => { + const query1 = Query.atPath(path('collection')); + const query2 = Query.atPath(path('collection')); + + return ( + client(0) + .userListens(query1) + .watchAcksFull(query1, 1000) + .expectEvents(query1, {}) + .client(1) + // Prevent client 0 from releasing its primary lease. + .disableNetwork() + .userListens(query1) + .expectEvents(query1, {}) + .client(0) + .disableNetwork() + .expectEvents(query1, { fromCache: true }) + .client(2) + .userListens(query1) + .expectEvents(query1, { fromCache: true }) + .userListens(query2) + .expectEvents(query2, { fromCache: true }) + ); + }); + + specTest( + 'New client becomes primary if no client has its network enabled', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + + return client(0) + .userListens(query) + .watchAcksFull(query, 1000) + .expectEvents(query, {}) + .client(1) + .userListens(query) + .expectEvents(query, {}) + .client(0) + .disableNetwork() + .expectEvents(query, { fromCache: true }) + .client(1) + .expectEvents(query, { fromCache: true }) + .client(2) + .expectListen(query, 'resume-token-1000') + .expectPrimaryState(true) + .watchAcksFull(query, 2000) + .client(0) + .expectEvents(query, {}) + .client(1) + .expectEvents(query, {}); + } + ); + + specTest( + "Secondary client's online state is ignored", + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 2000, { key: 'a' }); + + return ( + client(0) + .becomeVisible() + .client(1) + .userListens(query) + .client(0) + .expectListen(query) + .watchAcksFull(query, 1000) + .client(1) + .expectEvents(query, {}) + .disableNetwork() // Ignored since this is the secondary client. + .client(0) + .watchSends({ affects: [query] }, docA) + .watchSnapshots(2000) + .client(1) + .expectEvents(query, { added: [docA] }) + .client(0) + .disableNetwork() + // Client remains primary since all clients are offline. + .expectPrimaryState(true) + .client(1) + .expectEvents(query, { fromCache: true }) + .expectPrimaryState(false) + ); + } + ); + + specTest( + "Offline state doesn't persist if primary is shut down", + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + + return client(0) + .userListens(query) + .disableNetwork() + .expectEvents(query, { fromCache: true }) + .shutdown() + .client(1) + .userListens(query); // No event since the online state is 'Unknown'. + } + ); + + specTest( + 'Listen is re-listened to after primary tab failover', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + const docB = doc('collection/b', 2000, { key: 'b' }); + + return client(0) + .expectPrimaryState(true) + .client(1) + .userListens(query) + .client(0) + .expectListen(query) + .watchAcksFull(query, 1000, docA) + .client(1) + .expectEvents(query, { added: [docA] }) + .client(2) + .userListens(query) + .expectEvents(query, { added: [docA] }) + .client(0) + .shutdown() + .client(1) + .runTimer(TimerId.ClientMetadataRefresh) + .expectPrimaryState(true) + .expectListen(query, 'resume-token-1000') + .watchAcksFull(query, 2000, docB) + .expectEvents(query, { added: [docB] }) + .client(2) + .expectEvents(query, { added: [docB] }); + } + ); + + specTest('Listen is established in new primary tab', ['multi-client'], () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + const docB = doc('collection/b', 2000, { key: 'b' }); + + // Client 0 and Client 2 listen to the same query. When client 0 shuts + // down, client 1 becomes primary and takes ownership of a query it + // did not previously listen to. + return client(0) + .expectPrimaryState(true) + .userListens(query) + .watchAcksFull(query, 1000, docA) + .expectEvents(query, { added: [docA] }) + .client(1) // Start up and initialize the second client. + .client(2) + .userListens(query) + .expectEvents(query, { added: [docA] }) + .client(0) + .shutdown() + .client(1) + .runTimer(TimerId.ClientMetadataRefresh) + .expectPrimaryState(true) + .expectListen(query, 'resume-token-1000') + .watchAcksFull(query, 2000, docB) + .client(2) + .expectEvents(query, { added: [docB] }); + }); + + specTest('Query recovers after primary takeover', ['multi-client'], () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + const docB = doc('collection/b', 2000, { key: 'b' }); + const docC = doc('collection/c', 3000, { key: 'c' }); + + return ( + client(0) + .expectPrimaryState(true) + .userListens(query) + .watchAcksFull(query, 1000, docA) + .expectEvents(query, { added: [docA] }) + .client(1) + .userListens(query) + .expectEvents(query, { added: [docA] }) + .stealPrimaryLease() + .expectListen(query, 'resume-token-1000') + .watchAcksFull(query, 2000, docB) + .expectEvents(query, { added: [docB] }) + .client(0) + // Client 0 ignores all events until it transitions to secondary + .client(1) + .watchSends({ affects: [query] }, docC) + .watchSnapshots(3000) + .expectEvents(query, { added: [docC] }) + .client(0) + .runTimer(TimerId.ClientMetadataRefresh) + .expectPrimaryState(false) + .expectEvents(query, { added: [docB, docC] }) + ); + }); + + specTest( + 'Unresponsive primary ignores watch update', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + + return ( + client(0) + .expectPrimaryState(true) + .client(1) + .userListens(query) + .client(0) + .expectListen(query) + .client(1) + .stealPrimaryLease() + .client(0) + // Send a watch update to client 0, who is longer primary (but doesn't + // know it yet). The watch update gets ignored. + .watchAcksFull(query, 1000, docA) + .client(1) + .expectListen(query) + .watchAcksFull(query, 1000, docA) + .expectEvents(query, { added: [docA] }) + ); + } + ); + + specTest( + 'Listen is established in newly started primary', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + const docB = doc('collection/b', 2000, { key: 'b' }); + + // Client 0 executes a query on behalf of Client 1. When client 0 shuts + // down, client 2 starts up and becomes primary, taking ownership of the + // existing query. + return client(0) + .expectPrimaryState(true) + .client(1) + .userListens(query) + .client(0) + .expectListen(query) + .watchAcksFull(query, 1000, docA) + .client(1) + .expectEvents(query, { added: [docA] }) + .client(0) + .shutdown() + .client(2) + .expectPrimaryState(true) + .expectListen(query, 'resume-token-1000') + .watchAcksFull(query, 2000, docB) + .client(1) + .expectEvents(query, { added: [docB] }); + } + ); }); diff --git a/packages/firestore/test/unit/specs/orderby_spec.test.ts b/packages/firestore/test/unit/specs/orderby_spec.test.ts index 2fc2a9098c0..ca6469df1b2 100644 --- a/packages/firestore/test/unit/specs/orderby_spec.test.ts +++ b/packages/firestore/test/unit/specs/orderby_spec.test.ts @@ -56,4 +56,24 @@ describeSpec('OrderBy:', [], () => { .expectEvents(query1, { hasPendingWrites: true, added: [doc2b] }) ); }); + + specTest('orderBy applies to existing documents', [], () => { + const query = Query.atPath(path('collection')).addOrderBy( + orderBy('sort', 'asc') + ); + const docA = doc('collection/a', 0, { key: 'a', sort: 2 }); + const docB = doc('collection/b', 1001, { key: 'b', sort: 1 }); + + return spec() + .withGCEnabled(false) + .userListens(query) + .watchAcksFull(query, 1000, docA, docB) + .expectEvents(query, { added: [docB, docA] }) + .userUnlistens(query) + .watchRemoves(query) + .userListens(query, 'resume-token-1000') + .expectEvents(query, { added: [docB, docA], fromCache: true }) + .watchAcksFull(query, 1000) + .expectEvents(query, {}); + }); }); diff --git a/packages/firestore/test/unit/specs/perf_spec.test.ts b/packages/firestore/test/unit/specs/perf_spec.test.ts index ae636504c79..92907b0dc29 100644 --- a/packages/firestore/test/unit/specs/perf_spec.test.ts +++ b/packages/firestore/test/unit/specs/perf_spec.test.ts @@ -30,7 +30,9 @@ describeSpec( specTest('Insert a new document', [], () => { let steps = spec().withGCEnabled(false); for (let i = 0; i < STEP_COUNT; ++i) { - steps = steps.userSets(`collection/{i}`, { doc: i }).writeAcks(i); + steps = steps + .userSets(`collection/{i}`, { doc: i }) + .writeAcks(`collection/{i}`, i); } return steps; }); @@ -62,7 +64,7 @@ describeSpec( fromCache: true, hasPendingWrites: true }) - .writeAcks(++currentVersion) + .writeAcks(`collection/${i}`, ++currentVersion) .watchAcksFull(query, ++currentVersion, docRemote) .expectEvents(query, { metadata: [docRemote] }) .userUnlistens(query) @@ -106,7 +108,9 @@ describeSpec( let steps = spec().withGCEnabled(false); steps = steps.userSets(`collection/doc`, { v: 0 }); for (let i = 1; i <= STEP_COUNT; ++i) { - steps = steps.userPatches(`collection/doc`, { v: i }).writeAcks(i); + steps = steps + .userPatches(`collection/doc`, { v: i }) + .writeAcks(`collection/doc`, i); } return steps; }); @@ -137,7 +141,7 @@ describeSpec( fromCache: true, hasPendingWrites: true }) - .writeAcks(++currentVersion) + .writeAcks(`collection/doc`, ++currentVersion) .watchAcksFull(query, ++currentVersion, docRemote) .expectEvents(query, { metadata: [docRemote] }); @@ -157,7 +161,7 @@ describeSpec( modified: [docLocal], hasPendingWrites: true }) - .writeAcks(++currentVersion) + .writeAcks(`collection/doc`, ++currentVersion) .watchSends({ affects: [query] }, docRemote) .watchSnapshots(++currentVersion) .expectEvents(query, { metadata: [docRemote] }); diff --git a/packages/firestore/test/unit/specs/persistence_spec.test.ts b/packages/firestore/test/unit/specs/persistence_spec.test.ts index 4986cf09d1d..372438f6e09 100644 --- a/packages/firestore/test/unit/specs/persistence_spec.test.ts +++ b/packages/firestore/test/unit/specs/persistence_spec.test.ts @@ -18,17 +18,18 @@ import { Query } from '../../../src/core/query'; import { doc, path } from '../../util/helpers'; import { describeSpec, specTest } from './describe_spec'; -import { spec } from './spec_builder'; +import { client, spec } from './spec_builder'; +import { TimerId } from '../../../src/util/async_queue'; -describeSpec('Persistence:', ['persistence'], () => { +describeSpec('Persistence:', [], () => { specTest('Local mutations are persisted and re-sent', [], () => { return spec() .userSets('collection/key1', { foo: 'bar' }) .userSets('collection/key2', { baz: 'quu' }) .restart() .expectNumOutstandingWrites(2) - .writeAcks(1, { expectUserCallback: false }) - .writeAcks(2, { expectUserCallback: false }) + .writeAcks('collection/key1', 1, { expectUserCallback: false }) + .writeAcks('collection/key2', 2, { expectUserCallback: false }) .expectNumOutstandingWrites(0); }); @@ -97,7 +98,7 @@ describeSpec('Persistence:', ['persistence'], () => { .withGCEnabled(false) .userSets('collection/key', { foo: 'bar' }) // Normally the write would get GC'd from remote documents here. - .writeAcks(1000) + .writeAcks('collection/key', 1000) .userListens(query) // Version is 0 since we never received a server version via watch. .expectEvents(query, { @@ -116,11 +117,11 @@ describeSpec('Persistence:', ['persistence'], () => { .userSets('users/user1', { uid: 'user1', extra: true }) .changeUser(null) .expectNumOutstandingWrites(1) - .writeAcks(1000) + .writeAcks('users/anon', 1000) .changeUser('user1') .expectNumOutstandingWrites(2) - .writeAcks(2000) - .writeAcks(3000); + .writeAcks('users/user1', 2000) + .writeAcks('users/user1', 3000); }); specTest( @@ -136,12 +137,12 @@ describeSpec('Persistence:', ['persistence'], () => { .changeUser(null) .restart() .expectNumOutstandingWrites(1) - .writeAcks(1000, { expectUserCallback: false }) + .writeAcks('users/anon', 1000, { expectUserCallback: false }) .changeUser('user1') .restart() .expectNumOutstandingWrites(2) - .writeAcks(2000, { expectUserCallback: false }) - .writeAcks(3000, { expectUserCallback: false }); + .writeAcks('users/user1', 2000, { expectUserCallback: false }) + .writeAcks('users/user2', 3000, { expectUserCallback: false }); } ); @@ -181,4 +182,97 @@ describeSpec('Persistence:', ['persistence'], () => { }) ); }); + + specTest('Detects all active clients', ['multi-client'], () => { + return ( + client(0) + // While we don't verify the client's visibility in this test, the spec + // test framework requires an explicit action before setting an + // expectation. + .becomeHidden() + .expectNumActiveClients(1) + .client(1) + .becomeVisible() + .expectNumActiveClients(2) + ); + }); + + specTest('Single tab acquires primary lease', ['multi-client'], () => { + // This test simulates primary state handoff between two background tabs. + // With all instances in the background, the first active tab acquires + // ownership. + return client(0) + .becomeHidden() + .expectPrimaryState(true) + .client(1) + .becomeHidden() + .expectPrimaryState(false) + .client(0) + .shutdown() + .client(1) + .runTimer(TimerId.ClientMetadataRefresh) + .expectPrimaryState(true); + }); + + specTest('Foreground tab acquires primary lease', ['multi-client'], () => { + // This test verifies that in a multi-client scenario, a foreground tab + // takes precedence when a new primary client is elected. + return ( + client(0) + .becomeHidden() + .expectPrimaryState(true) + .client(1) + .becomeHidden() + .expectPrimaryState(false) + .client(2) + .becomeVisible() + .expectPrimaryState(false) + .client(0) + // Shutdown the client that is currently holding the primary lease. + .shutdown() + .client(1) + // Client 1 is in the background and doesn't grab the primary lease as + // client 2 is in the foreground. + .runTimer(TimerId.ClientMetadataRefresh) + .expectPrimaryState(false) + .client(2) + .runTimer(TimerId.ClientMetadataRefresh) + .expectPrimaryState(true) + ); + }); + + specTest('Primary lease bound to network state', ['multi-client'], () => { + return ( + client(0) + // If there is only a single tab, the online state is ignored and the + // tab is always primary + .expectPrimaryState(true) + .disableNetwork() + .expectPrimaryState(true) + .client(1) + .expectPrimaryState(false) + .client(0) + // If the primary tab is offline, and another tab becomes active, the + // primary tab releases its primary lease. + .runTimer(TimerId.ClientMetadataRefresh) + .expectPrimaryState(false) + .client(1) + .runTimer(TimerId.ClientMetadataRefresh) + .expectPrimaryState(true) + .disableNetwork() + // If all tabs are offline, the primary lease is retained. + .expectPrimaryState(true) + .client(0) + .enableNetwork() + .expectPrimaryState(false) + .client(1) + .runTimer(TimerId.ClientMetadataRefresh) + // The offline primary tab releases its lease since another tab is now + // online. + .expectPrimaryState(false) + .client(0) + .runTimer(TimerId.ClientMetadataRefresh) + .expectPrimaryState(true) + ); + }); }); diff --git a/packages/firestore/test/unit/specs/spec_builder.ts b/packages/firestore/test/unit/specs/spec_builder.ts index 523edf51969..165a6498a37 100644 --- a/packages/firestore/test/unit/specs/spec_builder.ts +++ b/packages/firestore/test/unit/specs/spec_builder.ts @@ -24,7 +24,11 @@ import { } from '../../../src/model/document'; import { DocumentKey } from '../../../src/model/document_key'; import { JsonObject } from '../../../src/model/field_value'; -import { mapRpcCodeFromCode } from '../../../src/remote/rpc_error'; +import { + isPermanentError, + mapCodeFromRpcCode, + mapRpcCodeFromCode +} from '../../../src/remote/rpc_error'; import { assert } from '../../../src/util/assert'; import { fail } from '../../../src/util/assert'; import { Code } from '../../../src/util/error'; @@ -42,10 +46,97 @@ import { SpecQueryFilter, SpecQueryOrderBy, SpecStep, - SpecWatchFilter + SpecWatchFilter, + SpecWriteAck, + SpecWriteFailure } from './spec_test_runner'; import { TimerId } from '../../../src/util/async_queue'; +// These types are used in a protected API by SpecBuilder and need to be +// exported. +export type QueryMap = { [query: string]: TargetId }; +export type LimboMap = { [key: string]: TargetId }; +export type ActiveTargetMap = { + [targetId: number]: { query: SpecQuery; resumeToken: string }; +}; + +/** + * Tracks the expected memory state of a client (e.g. the expected active watch + * targets based on userListens(), userUnlistens(), and watchRemoves() + * as well as the expectActiveTargets() and expectLimboDocs() expectations). + * + * Automatically keeping track of the active targets makes writing tests + * much simpler and the tests much easier to follow. + * + * Whenever the map changes, the expected state is automatically encoded in + * the tests. + */ +export class ClientMemoryState { + activeTargets: ActiveTargetMap; + queryMapping: QueryMap; + limboMapping: LimboMap; + + limboIdGenerator: TargetIdGenerator; + + constructor() { + this.reset(); + } + + /** Reset all internal memory state (as done during a client restart). */ + reset(): void { + this.queryMapping = {}; + this.limboMapping = {}; + this.activeTargets = {}; + this.limboIdGenerator = TargetIdGenerator.forSyncEngine(); + } + + /** + * Reset the internal limbo mapping (as done during a primary lease failover). + */ + resetLimboMapping(): void { + this.limboMapping = {}; + } +} + +/** + * Generates and provides consistent cross-tab target IDs for queries that are + * active in multiple tabs. + */ +class CachedTargetIdGenerator { + private queryMapping: QueryMap = {}; + private targetIdGenerator = TargetIdGenerator.forQueryCache(); + + /** + * Returns a cached target ID for the provided query, or a new ID if no + * target ID has ever been assigned. + */ + next(query: Query): TargetId { + if (objUtils.contains(this.queryMapping, query.canonicalId())) { + return this.queryMapping[query.canonicalId()]; + } + const targetId = this.targetIdGenerator.next(); + this.queryMapping[query.canonicalId()] = targetId; + return targetId; + } + + /** Returns the target ID for a query that is known to exist. */ + cachedId(query: Query): TargetId { + if (!objUtils.contains(this.queryMapping, query.canonicalId())) { + throw new Error("Target ID doesn't exists for query: " + query); + } + + return this.queryMapping[query.canonicalId()]; + } + + /** Remove the cached target ID for the provided query. */ + purge(query: Query): void { + if (!objUtils.contains(this.queryMapping, query.canonicalId())) { + throw new Error("Target ID doesn't exists for query: " + query); + } + + delete this.queryMapping[query.canonicalId()]; + } +} /** * Provides a high-level language to construct spec tests that can be exported * to the spec JSON format or be run as a spec test directly. @@ -54,31 +145,38 @@ import { TimerId } from '../../../src/util/async_queue'; * duplicate tests in every client. */ export class SpecBuilder { - private config: SpecConfig = { useGarbageCollection: true }; - private steps: SpecStep[] = []; + protected config: SpecConfig = { useGarbageCollection: true, numClients: 1 }; // currentStep is built up (in particular, expectations can be added to it) // until nextStep() is called to append it to steps. - private currentStep: SpecStep | null = null; - private queryMapping: { [query: string]: TargetId } = {}; - private limboMapping: { [key: string]: TargetId } = {}; + protected currentStep: SpecStep | null = null; - /** - * Tracks all expected active watch targets based on userListens(), - * userUnlistens(), and watchRemoves() steps and the expectActiveTargets() - * and expectLimboDocs() expectations. - * - * Automatically keeping track of the active targets makes writing tests - * much simpler and the tests much easier to follow. - * - * Whenever the map changes, the expected state is automatically encoded in - * the tests. - */ - private activeTargets: { - [targetId: number]: { query: SpecQuery; resumeToken: string }; - } = {}; + private steps: SpecStep[] = []; + + private queryIdGenerator = new CachedTargetIdGenerator(); - private queryIdGenerator: TargetIdGenerator = TargetIdGenerator.forLocalStore(); - private limboIdGenerator: TargetIdGenerator = TargetIdGenerator.forSyncEngine(); + private readonly currentClientState: ClientMemoryState = new ClientMemoryState(); + + // Accessor function that can be overridden to return a different + // `ClientMemoryState`. + protected get clientState(): ClientMemoryState { + return this.currentClientState; + } + + private get limboIdGenerator(): TargetIdGenerator { + return this.clientState.limboIdGenerator; + } + + private get queryMapping(): QueryMap { + return this.clientState.queryMapping; + } + + private get limboMapping(): LimboMap { + return this.clientState.limboMapping; + } + + private get activeTargets(): ActiveTargetMap { + return this.clientState.activeTargets; + } /** * Exports the spec steps as a JSON object that be used in the spec runner. @@ -98,7 +196,7 @@ export class SpecBuilder { } // Configures Garbage Collection behavior (on or off). Default is on. - withGCEnabled(gcEnabled: boolean): SpecBuilder { + withGCEnabled(gcEnabled: boolean): this { assert( !this.currentStep, 'withGCEnabled() must be called before all spec steps.' @@ -107,7 +205,7 @@ export class SpecBuilder { return this; } - userListens(query: Query, resumeToken?: string): SpecBuilder { + userListens(query: Query, resumeToken?: string): this { this.nextStep(); let targetId: TargetId = 0; @@ -118,7 +216,7 @@ export class SpecBuilder { targetId = this.queryMapping[query.canonicalId()]; } } else { - targetId = this.queryIdGenerator.next(); + targetId = this.queryIdGenerator.next(query); } this.queryMapping[query.canonicalId()] = targetId; @@ -137,7 +235,7 @@ export class SpecBuilder { * Registers a previously active target with the test expectations after a * stream disconnect. */ - restoreListen(query: Query, resumeToken: string): SpecBuilder { + restoreListen(query: Query, resumeToken: string): this { const targetId = this.queryMapping[query.canonicalId()]; if (isNullOrUndefined(targetId)) { @@ -157,7 +255,7 @@ export class SpecBuilder { return this; } - userUnlistens(query: Query): SpecBuilder { + userUnlistens(query: Query): this { this.nextStep(); if (!objUtils.contains(this.queryMapping, query.canonicalId())) { throw new Error('Unlistening to query not listened to: ' + query); @@ -165,6 +263,7 @@ export class SpecBuilder { const targetId = this.queryMapping[query.canonicalId()]; if (this.config.useGarbageCollection) { delete this.queryMapping[query.canonicalId()]; + this.queryIdGenerator.purge(query); } delete this.activeTargets[targetId]; this.currentStep = { @@ -174,7 +273,7 @@ export class SpecBuilder { return this; } - userSets(key: string, value: JsonObject): SpecBuilder { + userSets(key: string, value: JsonObject): this { this.nextStep(); this.currentStep = { userSet: [key, value] @@ -182,7 +281,7 @@ export class SpecBuilder { return this; } - userPatches(key: string, value: JsonObject): SpecBuilder { + userPatches(key: string, value: JsonObject): this { this.nextStep(); this.currentStep = { userPatch: [key, value] @@ -190,7 +289,7 @@ export class SpecBuilder { return this; } - userDeletes(key: string): SpecBuilder { + userDeletes(key: string): this { this.nextStep(); this.currentStep = { userDelete: key @@ -198,19 +297,37 @@ export class SpecBuilder { return this; } - runTimer(timerId: TimerId): SpecBuilder { + // PORTING NOTE: Only used by web multi-tab tests. + becomeHidden(): this { + this.nextStep(); + this.currentStep = { + applyClientState: { visibility: 'hidden' } + }; + return this; + } + + // PORTING NOTE: Only used by web multi-tab tests. + becomeVisible(): this { + this.nextStep(); + this.currentStep = { + applyClientState: { visibility: 'visible' } + }; + return this; + } + + runTimer(timerId: TimerId): this { this.nextStep(); this.currentStep = { runTimer: timerId }; return this; } - changeUser(uid: string | null): SpecBuilder { + changeUser(uid: string | null): this { this.nextStep(); this.currentStep = { changeUser: uid }; return this; } - disableNetwork(): SpecBuilder { + disableNetwork(): this { this.nextStep(); this.currentStep = { enableNetwork: false, @@ -222,7 +339,7 @@ export class SpecBuilder { return this; } - enableNetwork(): SpecBuilder { + enableNetwork(): this { this.nextStep(); this.currentStep = { enableNetwork: true @@ -230,7 +347,7 @@ export class SpecBuilder { return this; } - restart(): SpecBuilder { + restart(): this { this.nextStep(); this.currentStep = { restart: true, @@ -239,24 +356,34 @@ export class SpecBuilder { limboDocs: [] } }; + // Reset our mappings / target ids since all existing listens will be + // forgotten. + this.clientState.reset(); + return this; + } + shutdown(): this { + this.nextStep(); + this.currentStep = { + shutdown: true, + stateExpect: { + activeTargets: {}, + limboDocs: [] + } + }; // Reset our mappings / target ids since all existing listens will be - // forgotten - this.queryMapping = {}; - this.limboMapping = {}; - this.activeTargets = {}; - this.queryIdGenerator = TargetIdGenerator.forLocalStore(); - this.limboIdGenerator = TargetIdGenerator.forSyncEngine(); + // forgotten. + this.clientState.reset(); return this; } /** Overrides the currently expected set of active targets. */ expectActiveTargets( ...targets: Array<{ query: Query; resumeToken: string }> - ): SpecBuilder { + ): this { this.assertStep('Active target expectation requires previous step'); const currentStep = this.currentStep!; - this.activeTargets = {}; + this.clientState.activeTargets = {}; targets.forEach(({ query, resumeToken }) => { this.activeTargets[this.getTargetId(query)] = { query: SpecBuilder.queryToSpec(query), @@ -274,7 +401,7 @@ export class SpecBuilder { * Expects a document to be in limbo. A targetId is assigned if it's not in * limbo yet. */ - expectLimboDocs(...keys: DocumentKey[]): SpecBuilder { + expectLimboDocs(...keys: DocumentKey[]): this { this.assertStep('Limbo expectation requires previous step'); const currentStep = this.currentStep!; @@ -310,10 +437,7 @@ export class SpecBuilder { * with no document for NoDocument. This is translated into normal watch * messages. */ - ackLimbo( - version: TestSnapshotVersion, - doc: Document | NoDocument - ): SpecBuilder { + ackLimbo(version: TestSnapshotVersion, doc: Document | NoDocument): this { const query = Query.atPath(doc.key.path); this.watchAcks(query); if (doc instanceof Document) { @@ -333,7 +457,7 @@ export class SpecBuilder { * with either a document or with no document for NoDocument. This is * translated into normal watch messages. */ - watchRemovesLimboTarget(doc: Document | NoDocument): SpecBuilder { + watchRemovesLimboTarget(doc: Document | NoDocument): this { const query = Query.atPath(doc.key.path); this.watchRemoves(query); return this; @@ -342,44 +466,64 @@ export class SpecBuilder { /** * Acks a write with a version and optional additional options. * - * expectUserCallback defaults to true if options are omitted. + * expectUserCallback defaults to true if omitted. */ writeAcks( + doc: string, version: TestSnapshotVersion, - options?: { - expectUserCallback: boolean; - } - ): SpecBuilder { + options?: { expectUserCallback?: boolean; keepInQueue?: boolean } + ): this { this.nextStep(); - this.currentStep = { - writeAck: { - version, - expectUserCallback: options ? options.expectUserCallback : true - } - }; - return this; + options = options || {}; + + const writeAck: SpecWriteAck = { version }; + if (options.keepInQueue) { + writeAck.keepInQueue = true; + } + this.currentStep = { writeAck }; + + if (options.expectUserCallback !== false) { + return this.expectUserCallbacks({ acknowledged: [doc] }); + } else { + return this; + } } /** * Fails a write with an error and optional additional options. * - * expectUserCallback defaults to true if options are omitted. + * expectUserCallback defaults to true if omitted. */ failWrite( - err: RpcError, - options?: { expectUserCallback: boolean } - ): SpecBuilder { + doc: string, + error: RpcError, + options?: { expectUserCallback?: boolean; keepInQueue?: boolean } + ): this { this.nextStep(); - this.currentStep = { - failWrite: { - error: err, - expectUserCallback: options ? options.expectUserCallback : true - } - }; - return this; + options = options || {}; + + // If this is a permanent error, the write is not expected to be sent + // again. + const isPermanentFailure = isPermanentError(mapCodeFromRpcCode(error.code)); + const keepInQueue = + options.keepInQueue !== undefined + ? options.keepInQueue + : !isPermanentFailure; + + const failWrite: SpecWriteFailure = { error }; + if (keepInQueue) { + failWrite.keepInQueue = true; + } + this.currentStep = { failWrite }; + + if (options.expectUserCallback !== false) { + return this.expectUserCallbacks({ rejected: [doc] }); + } else { + return this; + } } - watchAcks(query: Query): SpecBuilder { + watchAcks(query: Query): this { this.nextStep(); this.currentStep = { watchAck: [this.getTargetId(query)] @@ -392,7 +536,7 @@ export class SpecBuilder { // Eventually we want to make the model more generic so we can add resume // tokens in other places. // TODO(b/37254270): Handle global resume tokens - watchCurrents(query: Query, resumeToken: string): SpecBuilder { + watchCurrents(query: Query, resumeToken: string): this { this.nextStep(); this.currentStep = { watchCurrent: [[this.getTargetId(query)], resumeToken] @@ -400,7 +544,7 @@ export class SpecBuilder { return this; } - watchRemoves(query: Query, cause?: RpcError): SpecBuilder { + watchRemoves(query: Query, cause?: RpcError): this { this.nextStep(); this.currentStep = { watchRemove: { targetIds: [this.getTargetId(query)], cause } @@ -417,7 +561,7 @@ export class SpecBuilder { watchSends( targets: { affects?: Query[]; removed?: Query[] }, ...docs: MaybeDocument[] - ): SpecBuilder { + ): this { this.nextStep(); const affects = targets.affects && @@ -443,7 +587,7 @@ export class SpecBuilder { return this; } - watchRemovesDoc(key: DocumentKey, ...targets: Query[]): SpecBuilder { + watchRemovesDoc(key: DocumentKey, ...targets: Query[]): this { this.nextStep(); this.currentStep = { watchEntity: { @@ -454,7 +598,7 @@ export class SpecBuilder { return this; } - watchFilters(queries: Query[], ...docs: DocumentKey[]): SpecBuilder { + watchFilters(queries: Query[], ...docs: DocumentKey[]): this { this.nextStep(); const targetIds = queries.map(query => { return this.getTargetId(query); @@ -472,7 +616,7 @@ export class SpecBuilder { return this; } - watchResets(...queries: Query[]): SpecBuilder { + watchResets(...queries: Query[]): this { this.nextStep(); const targetIds = queries.map(query => this.getTargetId(query)); this.currentStep = { @@ -485,7 +629,7 @@ export class SpecBuilder { version: TestSnapshotVersion, targets?: Query[], resumeToken?: string - ): SpecBuilder { + ): this { this.nextStep(); const targetIds = targets && targets.map(query => this.getTargetId(query)); this.currentStep = { @@ -498,7 +642,7 @@ export class SpecBuilder { query: Query, version: TestSnapshotVersion, ...docs: Document[] - ): SpecBuilder { + ): this { this.watchAcks(query); this.watchSends({ affects: [query] }, ...docs); this.watchCurrents(query, 'resume-token-' + version); @@ -506,10 +650,7 @@ export class SpecBuilder { return this; } - watchStreamCloses( - error: Code, - opts?: { runBackoffTimer: boolean } - ): SpecBuilder { + watchStreamCloses(error: Code, opts?: { runBackoffTimer: boolean }): this { if (!opts) { opts = { runBackoffTimer: true }; } @@ -527,6 +668,29 @@ export class SpecBuilder { return this; } + expectUserCallbacks(docs: { + acknowledged?: string[]; + rejected?: string[]; + }): this { + this.assertStep('Expectations require previous step'); + const currentStep = this.currentStep!; + currentStep.stateExpect = currentStep.stateExpect || {}; + currentStep.stateExpect.userCallbacks = currentStep.stateExpect + .userCallbacks || { acknowledgedDocs: [], rejectedDocs: [] }; + + if (docs.acknowledged) { + currentStep.stateExpect.userCallbacks.acknowledgedDocs.push( + ...docs.acknowledged + ); + } + + if (docs.rejected) { + currentStep.stateExpect.userCallbacks.rejectedDocs.push(...docs.rejected); + } + + return this; + } + expectEvents( query: Query, events: { @@ -538,7 +702,7 @@ export class SpecBuilder { metadata?: Document[]; errorCode?: Code; } - ): SpecBuilder { + ): this { this.assertStep('Expectations require previous step'); const currentStep = this.currentStep!; if (!currentStep.expect) { @@ -562,11 +726,52 @@ export class SpecBuilder { return this; } + /** Registers a query that is active in another tab. */ + expectListen(query: Query, resumeToken?: string): this { + this.assertStep('Expectations require previous step'); + + const targetId = this.queryIdGenerator.cachedId(query); + this.queryMapping[query.canonicalId()] = targetId; + + this.activeTargets[targetId] = { + query: SpecBuilder.queryToSpec(query), + resumeToken: resumeToken || '' + }; + + const currentStep = this.currentStep!; + currentStep.stateExpect = currentStep.stateExpect || {}; + currentStep.stateExpect.activeTargets = objUtils.shallowCopy( + this.activeTargets + ); + return this; + } + + /** Removes a query that is no longer active in any tab. */ + expectUnlisten(query: Query): this { + this.assertStep('Expectations require previous step'); + + const targetId = this.queryMapping[query.canonicalId()]; + + if (this.config.useGarbageCollection) { + delete this.queryMapping[query.canonicalId()]; + this.queryIdGenerator.purge(query); + } + + delete this.activeTargets[targetId]; + + const currentStep = this.currentStep!; + currentStep.stateExpect = currentStep.stateExpect || {}; + currentStep.stateExpect.activeTargets = objUtils.shallowCopy( + this.activeTargets + ); + return this; + } + /** * Verifies the total number of requests sent to the write backend since test * initialization. */ - expectWriteStreamRequestCount(num: number): SpecBuilder { + expectWriteStreamRequestCount(num: number): this { this.assertStep('Expectations require previous step'); const currentStep = this.currentStep!; currentStep.stateExpect = currentStep.stateExpect || {}; @@ -578,7 +783,7 @@ export class SpecBuilder { * Verifies the total number of requests sent to the watch backend since test * initialization. */ - expectWatchStreamRequestCount(num: number): SpecBuilder { + expectWatchStreamRequestCount(num: number): this { this.assertStep('Expectations require previous step'); const currentStep = this.currentStep!; currentStep.stateExpect = currentStep.stateExpect || {}; @@ -586,7 +791,7 @@ export class SpecBuilder { return this; } - expectNumOutstandingWrites(num: number): SpecBuilder { + expectNumOutstandingWrites(num: number): this { this.assertStep('Expectations require previous step'); const currentStep = this.currentStep!; currentStep.stateExpect = currentStep.stateExpect || {}; @@ -594,6 +799,22 @@ export class SpecBuilder { return this; } + expectNumActiveClients(num: number): this { + this.assertStep('Expectations require previous step'); + const currentStep = this.currentStep!; + currentStep.stateExpect = currentStep.stateExpect || {}; + currentStep.stateExpect.numActiveClients = num; + return this; + } + + expectPrimaryState(isPrimary: boolean): this { + this.assertStep('Expectations requires previous step'); + const currentStep = this.currentStep!; + currentStep.stateExpect = currentStep.stateExpect || {}; + currentStep.stateExpect.isPrimary = isPrimary; + return this; + } + private static queryToSpec(query: Query): SpecQuery { // TODO(dimond): full query support const spec: SpecQuery = { path: query.path.canonicalString() }; @@ -647,7 +868,7 @@ export class SpecBuilder { return key.path.canonicalString(); } - private nextStep(): void { + protected nextStep(): void { if (this.currentStep !== null) { this.steps.push(this.currentStep); this.currentStep = null; @@ -677,6 +898,86 @@ export class SpecBuilder { } } +/** + * SpecBuilder that supports serialized interactions between different clients. + * + * Use `client(clientIndex)` to switch between clients. + */ +// PORTING NOTE: Only used by web multi-tab tests. +export class MultiClientSpecBuilder extends SpecBuilder { + private activeClientIndex = -1; + private clientStates: ClientMemoryState[] = []; + + protected get clientState(): ClientMemoryState { + if (!this.clientStates[this.activeClientIndex]) { + this.clientStates[this.activeClientIndex] = new ClientMemoryState(); + } + return this.clientStates[this.activeClientIndex]; + } + + client(clientIndex: number): MultiClientSpecBuilder { + // Since `currentStep` is fully self-contained and does not rely on previous + // state, we don't need to use a different SpecBuilder instance for each + // client. + this.nextStep(); + this.currentStep = { + drainQueue: true + }; + + this.activeClientIndex = clientIndex; + this.config.numClients = Math.max( + this.config.numClients, + this.activeClientIndex + 1 + ); + + return this; + } + + /** + * Take the primary lease, even if another client has already obtained the + * lease. + */ + stealPrimaryLease(): this { + this.nextStep(); + this.currentStep = { + applyClientState: { + primary: true + }, + stateExpect: { + isPrimary: true + } + }; + + // HACK: SyncEngine resets its limbo mapping when it gains the primary + // lease. The SpecTests need to also clear their mapping, but when we parse + // the spec tests, we don't know when the primary lease transition happens. + // It is likely going to happen right after `stealPrimaryLease`, so we are + // clearing the limbo mapping here. + this.clientState.resetLimboMapping(); + + return this; + } + + protected nextStep(): void { + if (this.currentStep !== null) { + this.currentStep.clientIndex = this.activeClientIndex; + } + super.nextStep(); + } +} + +/** Starts a new single-client SpecTest. */ export function spec(): SpecBuilder { return new SpecBuilder(); } + +/** Starts a new multi-client SpecTest. */ +// PORTING NOTE: Only used by web multi-tab tests. +export function client( + num: number, + withGcEnabled?: boolean +): MultiClientSpecBuilder { + const specBuilder = new MultiClientSpecBuilder(); + specBuilder.withGCEnabled(withGcEnabled === true); + return specBuilder.client(num); +} diff --git a/packages/firestore/test/unit/specs/spec_test_runner.ts b/packages/firestore/test/unit/specs/spec_test_runner.ts index fd7f5a7d245..7f68c50836c 100644 --- a/packages/firestore/test/unit/specs/spec_test_runner.ts +++ b/packages/firestore/test/unit/specs/spec_test_runner.ts @@ -29,6 +29,7 @@ import { SnapshotVersion } from '../../../src/core/snapshot_version'; import { SyncEngine } from '../../../src/core/sync_engine'; import { OnlineState, + OnlineStateSource, ProtoByteString, TargetId } from '../../../src/core/types'; @@ -50,16 +51,16 @@ import { DocumentOptions } from '../../../src/model/document'; import { DocumentKey } from '../../../src/model/document_key'; import { JsonObject } from '../../../src/model/field_value'; import { Mutation } from '../../../src/model/mutation'; -import { emptyByteString } from '../../../src/platform/platform'; +import { + emptyByteString, + PlatformSupport +} from '../../../src/platform/platform'; import { Connection, Stream } from '../../../src/remote/connection'; import { Datastore } from '../../../src/remote/datastore'; import { ExistenceFilter } from '../../../src/remote/existence_filter'; import { WriteRequest } from '../../../src/remote/persistent_stream'; import { RemoteStore } from '../../../src/remote/remote_store'; -import { - isPermanentError, - mapCodeFromRpcCode -} from '../../../src/remote/rpc_error'; +import { mapCodeFromRpcCode } from '../../../src/remote/rpc_error'; import { JsonProtoSerializer } from '../../../src/remote/serializer'; import { StreamBridge } from '../../../src/remote/stream_bridge'; import { @@ -90,6 +91,19 @@ import { version, expectFirestoreError } from '../../util/helpers'; +import { + ClientId, + MemorySharedClientState, + SharedClientState, + WebStorageSharedClientState +} from '../../../src/local/shared_client_state'; +import { + createOrUpgradeDb, + DbOwner, + DbOwnerKey, + SCHEMA_VERSION +} from '../../../src/local/indexeddb_schema'; +import { TestPlatform, SharedFakeWebStorage } from '../../util/test_platform'; class MockConnection implements Connection { watchStream: StreamBridge< @@ -128,13 +142,6 @@ class MockConnection implements Connection { /** A Deferred that is resolved once watch opens. */ watchOpen = new Deferred(); - reset(): void { - this.watchStreamRequestCount = 0; - this.writeStreamRequestCount = 0; - this.earlyWrites = []; - this.activeTargets = []; - } - invokeRPC(rpcName: string, request: Req): never { throw new Error('Not implemented!'); } @@ -315,19 +322,41 @@ class EventAggregator implements Observer { } } -interface OutstandingWrite { - mutations: Mutation[]; - userCallback: Deferred; +/** + * FIFO queue that tracks all outstanding mutations for a single test run. + * As these mutations are shared among the set of active clients, any client can + * add or retrieve mutations. + */ +// PORTING NOTE: Multi-tab only. +class SharedWriteTracker { + private writes: Mutation[][] = []; + + push(write: Mutation[]): void { + this.writes.push(write); + } + + peek(): Mutation[] { + assert(this.writes.length > 0, 'No pending mutations'); + return this.writes[0]; + } + + shift(): Mutation[] { + assert(this.writes.length > 0, 'No pending mutations'); + return this.writes.shift()!; + } } abstract class TestRunner { + protected queue: AsyncQueue; + private connection: MockConnection; private eventManager: EventManager; private syncEngine: SyncEngine; - private queue: AsyncQueue; private eventList: QueryEvent[] = []; - private outstandingWrites: OutstandingWrite[] = []; + private acknowledgedDocs: string[]; + private rejectedDocs: string[]; + private queryListeners = new ObjectMap(q => q.canonicalId() ); @@ -341,46 +370,63 @@ abstract class TestRunner { private localStore: LocalStore; private remoteStore: RemoteStore; private persistence: Persistence; + private sharedClientState: SharedClientState; private useGarbageCollection: boolean; + private numClients: number; private databaseInfo: DatabaseInfo; - private user = User.UNAUTHENTICATED; + protected user = User.UNAUTHENTICATED; + protected clientId: ClientId; + + private started = false; private serializer: JsonProtoSerializer; - constructor(private readonly name: string, config: SpecConfig) { + constructor( + protected readonly platform: TestPlatform, + private sharedWrites: SharedWriteTracker, + clientIndex: number, + config: SpecConfig + ) { + this.clientId = `client${clientIndex}`; this.databaseInfo = new DatabaseInfo( new DatabaseId('project'), 'persistenceKey', 'host', false ); + + // TODO(mrschmidt): During client startup in `firestore_client`, we block + // the AsyncQueue from executing any operation. We should mimic this in the + // setup of the spec tests. + this.queue = new AsyncQueue(); this.serializer = new JsonProtoSerializer(this.databaseInfo.databaseId, { useProto3Json: true }); this.useGarbageCollection = config.useGarbageCollection; - - this.queue = new AsyncQueue(); + this.numClients = config.numClients; this.expectedLimboDocs = []; this.expectedActiveTargets = {}; + this.acknowledgedDocs = []; + this.rejectedDocs = []; } async start(): Promise { - this.persistence = this.getPersistence(this.serializer); - await this.persistence.start(); + this.persistence = await this.initPersistence(this.serializer); await this.init(); } - async init(): Promise { + private async init(): Promise { const garbageCollector = this.getGarbageCollector(); + this.sharedClientState = this.getSharedClientState(); this.localStore = new LocalStore( this.persistence, this.user, - garbageCollector + garbageCollector, + this.sharedClientState ); - await this.localStore.start(); this.connection = new MockConnection(this.queue); this.datastore = new Datastore( @@ -389,28 +435,49 @@ abstract class TestRunner { new EmptyCredentialsProvider(), this.serializer ); - const onlineStateChangedHandler = (onlineState: OnlineState) => { - this.syncEngine.applyOnlineStateChange(onlineState); - this.eventManager.applyOnlineStateChange(onlineState); + const remoteStoreOnlineStateChangedHandler = (onlineState: OnlineState) => { + this.syncEngine.applyOnlineStateChange( + onlineState, + OnlineStateSource.RemoteStore + ); + }; + const sharedClientStateOnlineStateChangedHandler = ( + onlineState: OnlineState + ) => { + this.syncEngine.applyOnlineStateChange( + onlineState, + OnlineStateSource.SharedClientState + ); }; this.remoteStore = new RemoteStore( this.localStore, this.datastore, this.queue, - onlineStateChangedHandler + remoteStoreOnlineStateChangedHandler ); - await this.remoteStore.start(); - this.syncEngine = new SyncEngine( this.localStore, this.remoteStore, + this.sharedClientState, this.user ); - // Setup wiring between sync engine and remote store + // Set up wiring between sync engine and other components this.remoteStore.syncEngine = this.syncEngine; + this.sharedClientState.syncEngine = this.syncEngine; + this.sharedClientState.onlineStateHandler = sharedClientStateOnlineStateChangedHandler; this.eventManager = new EventManager(this.syncEngine); + + await this.localStore.start(); + await this.sharedClientState.start(); + await this.remoteStore.start(); + + await this.persistence.setPrimaryStateListener(isPrimary => + this.syncEngine.applyPrimaryState(isPrimary) + ); + + this.started = true; } private getGarbageCollector(): GarbageCollector { @@ -419,29 +486,33 @@ abstract class TestRunner { : new NoOpGarbageCollector(); } - protected abstract getPersistence( + protected abstract initPersistence( serializer: JsonProtoSerializer - ): Persistence; - protected abstract destroyPersistence(): Promise; + ): Promise; + + protected abstract getSharedClientState(): SharedClientState; + + get isPrimaryClient(): boolean { + return this.syncEngine.isPrimaryClient; + } async shutdown(): Promise { await this.queue.enqueue(async () => { - await this.remoteStore.shutdown(); - await this.persistence.shutdown(/* deleteData= */ true); - await this.destroyPersistence(); + if (this.started) { + await this.doShutdown(); + } }); } - run(steps: SpecStep[]): Promise { - // tslint:disable-next-line:no-console - console.log('Running spec: ' + this.name); - return sequence(steps, async step => { - await this.doStep(step); - await this.queue.drain(); - this.validateStepExpectations(step.expect!); - this.validateStateExpectations(step.stateExpect!); - this.eventList = []; - }); + /** Runs a single SpecStep on this runner. */ + async run(step: SpecStep): Promise { + await this.doStep(step); + await this.queue.drain(); + this.validateStepExpectations(step.expect!); + await this.validateStateExpectations(step.stateExpect!); + this.eventList = []; + this.rejectedDocs = []; + this.acknowledgedDocs = []; } private doStep(step: SpecStep): Promise { @@ -477,13 +548,19 @@ abstract class TestRunner { return this.doFailWrite(step.failWrite!); } else if ('runTimer' in step) { return this.doRunTimer(step.runTimer!); + } else if ('drainQueue' in step) { + return this.doDrainQueue(); } else if ('enableNetwork' in step) { return step.enableNetwork! ? this.doEnableNetwork() : this.doDisableNetwork(); } else if ('restart' in step) { - assert(step.restart!, 'Restart cannot be false'); return this.doRestart(); + } else if ('shutdown' in step) { + return this.doShutdown(); + } else if ('applyClientState' in step) { + // PORTING NOTE: Only used by web multi-tab tests. + return this.doApplyClientState(step.applyClientState!); } else if ('changeUser' in step) { return this.doChangeUser(step.changeUser!); } else { @@ -522,8 +599,10 @@ abstract class TestRunner { ); } - // Open should always have happened after a listen - await this.connection.waitForWatchOpen(); + if (this.isPrimaryClient) { + // Open should always have happened after a listen + await this.connection.waitForWatchOpen(); + } } private async doUnlisten(listenSpec: SpecUserUnlisten): Promise { @@ -550,11 +629,19 @@ abstract class TestRunner { return this.doMutations([deleteMutation(key)]); } - private async doMutations(mutations: Mutation[]): Promise { - const userCallback = new Deferred(); - this.outstandingWrites.push({ mutations, userCallback }); + private doMutations(mutations: Mutation[]): Promise { + const documentKeys = mutations.map(val => val.key.path.toString()); + const syncEngineCallback = new Deferred(); + + syncEngineCallback.promise.then( + () => this.acknowledgedDocs.push(...documentKeys), + () => this.rejectedDocs.push(...documentKeys) + ); + + this.sharedWrites.push(mutations); + return this.queue.enqueue(() => { - return this.syncEngine.write(mutations, userCallback); + return this.syncEngine.write(mutations, syncEngineCallback); }); } @@ -725,12 +812,11 @@ abstract class TestRunner { private doWriteAck(writeAck: SpecWriteAck): Promise { const updateTime = this.serializer.toVersion(version(writeAck.version)); - const nextWrite = this.outstandingWrites.shift()!; - return this.validateNextWriteRequest(nextWrite.mutations).then(() => { + const nextMutation = writeAck.keepInQueue + ? this.sharedWrites.peek() + : this.sharedWrites.shift(); + return this.validateNextWriteRequest(nextMutation).then(() => { this.connection.ackWrite(updateTime, [{ updateTime }]); - if (writeAck.expectUserCallback) { - return nextWrite.userCallback.promise; - } }); } @@ -740,25 +826,11 @@ abstract class TestRunner { mapCodeFromRpcCode(specError.code), specError.message ); - const nextWrite = this.outstandingWrites.shift()!; - return this.validateNextWriteRequest(nextWrite.mutations).then(() => { - // If this is not a permanent error, the write is expected to be sent - // again. - if (!isPermanentError(error.code)) { - this.outstandingWrites.unshift(nextWrite); - } - + const nextMutation = writeFailure.keepInQueue + ? this.sharedWrites.peek() + : this.sharedWrites.shift(); + return this.validateNextWriteRequest(nextMutation).then(() => { this.connection.failWrite(error); - if (writeFailure.expectUserCallback) { - return nextWrite.userCallback.promise.then( - () => { - fail('write should have failed'); - }, - err => { - expect(err).not.to.be.null; - } - ); - } }); } @@ -774,11 +846,25 @@ abstract class TestRunner { // Make sure to execute all writes that are currently queued. This allows us // to assert on the total number of requests sent before shutdown. await this.remoteStore.fillWritePipeline(); - await this.remoteStore.disableNetwork(); + await this.syncEngine.disableNetwork(); + } + + private async doDrainQueue(): Promise { + await this.queue.drain(); } private async doEnableNetwork(): Promise { - await this.remoteStore.enableNetwork(); + await this.syncEngine.enableNetwork(); + } + + private async doShutdown(): Promise { + await this.remoteStore.shutdown(); + await this.sharedClientState.shutdown(); + // We don't delete the persisted data here since multi-clients may still + // be accessing it. Instead, we manually remove it at the end of the + // test run. + await this.persistence.shutdown(/* deleteData= */ false); + this.started = false; } private async doRestart(): Promise { @@ -788,9 +874,20 @@ abstract class TestRunner { // We have to schedule the starts, otherwise we could end up with // interleaved events. - await this.queue.enqueue(async () => { - await this.init(); - }); + await this.queue.enqueue(() => this.init()); + } + + private async doApplyClientState(state: SpecClientState): Promise { + if (state.visibility) { + this.platform.raiseVisibilityEvent(state.visibility!); + } + + if (state.primary) { + await writeOwnerToIndexedDb(this.clientId); + await this.queue.runDelayedOperationsEarly(TimerId.ClientMetadataRefresh); + } + + return Promise.resolve(); } private doChangeUser(user: string | null): Promise { @@ -819,13 +916,19 @@ abstract class TestRunner { } } - private validateStateExpectations(expectation: StateExpectation): void { + private async validateStateExpectations( + expectation: StateExpectation + ): Promise { if (expectation) { if ('numOutstandingWrites' in expectation) { expect(this.remoteStore.outstandingWrites()).to.equal( expectation.numOutstandingWrites ); } + if ('numActiveClients' in expectation) { + const activeClients = await this.persistence.getActiveClients(); + expect(activeClients.length).to.equal(expectation.numActiveClients); + } if ('writeStreamRequestCount' in expectation) { expect(this.connection.writeStreamRequestCount).to.equal( expectation.writeStreamRequestCount @@ -842,13 +945,37 @@ abstract class TestRunner { if ('activeTargets' in expectation) { this.expectedActiveTargets = expectation.activeTargets!; } + if ('isPrimary' in expectation) { + expect(this.isPrimaryClient).to.eq(expectation.isPrimary!, 'isPrimary'); + } + } + + if (expectation && expectation.userCallbacks) { + expect(this.acknowledgedDocs).to.have.members( + expectation.userCallbacks.acknowledgedDocs + ); + expect(this.rejectedDocs).to.have.members( + expectation.userCallbacks.rejectedDocs + ); + } else { + expect(this.acknowledgedDocs).to.be.empty; + expect(this.rejectedDocs).to.be.empty; + } + + if (this.numClients === 1) { + expect(this.isPrimaryClient).to.eq(true, 'isPrimary'); } - // Always validate that the expected limbo docs match the actual limbo docs - this.validateLimboDocs(); - // Always validate that the expected active targets match the actual active - // targets - this.validateActiveTargets(); + // Clients don't reset their limbo docs on shutdown, so any validation will + // likely fail. + if (this.started) { + // Always validate that the expected limbo docs match the actual limbo + // docs + this.validateLimboDocs(); + // Always validate that the expected active targets match the actual + // active targets + await this.validateActiveTargets(); + } } private validateLimboDocs(): void { @@ -874,7 +1001,22 @@ abstract class TestRunner { ); } - private validateActiveTargets(): void { + private async validateActiveTargets(): Promise { + if (!this.isPrimaryClient) { + expect(this.connection.activeTargets).to.be.empty; + return; + } + + // In multi-tab mode, we cannot rely on the `waitForWatchOpen` call in + // `doUserListen` since primary tabs may execute queries from other tabs + // without any direct user interaction. + // TODO(multitab): Refactor so this is only executed after primary tab + // change + if (!obj.isEmpty(this.expectedActiveTargets)) { + await this.connection.waitForWatchOpen(); + await this.queue.drain(); + } + const actualTargets = obj.shallowCopy(this.connection.activeTargets); obj.forEachNumber(this.expectedActiveTargets, (targetId, expected) => { expect(obj.contains(actualTargets, targetId)).to.equal( @@ -992,12 +1134,16 @@ abstract class TestRunner { } class MemoryTestRunner extends TestRunner { - protected getPersistence(serializer: JsonProtoSerializer): MemoryPersistence { - return new MemoryPersistence(); + protected getSharedClientState(): SharedClientState { + return new MemorySharedClientState(); } - protected async destroyPersistence(): Promise { - // Nothing to do. + protected async initPersistence( + serializer: JsonProtoSerializer + ): Promise { + const persistence = new MemoryPersistence(this.clientId); + await persistence.start(); + return persistence; } } @@ -1006,18 +1152,32 @@ class MemoryTestRunner extends TestRunner { * enabled for the platform. */ class IndexedDbTestRunner extends TestRunner { - static TEST_DB_NAME = 'specs'; + static TEST_DB_NAME = 'firestore/[DEFAULT]/specs'; + protected getSharedClientState(): SharedClientState { + return new WebStorageSharedClientState( + this.queue, + this.platform, + IndexedDbTestRunner.TEST_DB_NAME, + this.clientId, + this.user + ); + } - protected getPersistence( + protected async initPersistence( serializer: JsonProtoSerializer - ): IndexedDbPersistence { - return new IndexedDbPersistence( + ): Promise { + const persistence = new IndexedDbPersistence( IndexedDbTestRunner.TEST_DB_NAME, + this.clientId, + this.platform, + this.queue, serializer ); + await persistence.start(/*synchronizeTabs=*/ true); + return persistence; } - protected destroyPersistence(): Promise { + static destroyPersistence(): Promise { return SimpleDb.delete( IndexedDbTestRunner.TEST_DB_NAME + IndexedDbPersistence.MAIN_DATABASE ); @@ -1035,17 +1195,63 @@ export async function runSpec( config: SpecConfig, steps: SpecStep[] ): Promise { - let runner: TestRunner; - if (usePersistence) { - runner = new IndexedDbTestRunner(name, config); - } else { - runner = new MemoryTestRunner(name, config); - } - await runner.start(); + // tslint:disable-next-line:no-console + console.log('Running spec: ' + name); + + const sharedMockStorage = new SharedFakeWebStorage(); + + // PORTING NOTE: Non multi-client SDKs only support a single test runner. + const runners: TestRunner[] = []; + const outstandingMutations = new SharedWriteTracker(); + + const ensureRunner = async clientIndex => { + if (!runners[clientIndex]) { + const platform = new TestPlatform( + PlatformSupport.getPlatform(), + sharedMockStorage + ); + if (usePersistence) { + runners[clientIndex] = new IndexedDbTestRunner( + platform, + outstandingMutations, + clientIndex, + config + ); + } else { + runners[clientIndex] = new MemoryTestRunner( + platform, + outstandingMutations, + clientIndex, + config + ); + } + await runners[clientIndex].start(); + } + return runners[clientIndex]; + }; + + let lastStep = null; + let count = 0; try { - await runner.run(steps); + await sequence(steps, async step => { + ++count; + lastStep = step; + return ensureRunner(step.clientIndex || 0).then(runner => + runner.run(step) + ); + }); + } catch (err) { + console.warn( + `Spec test failed at step ${count}: ${JSON.stringify(lastStep)}` + ); + throw err; } finally { - await runner.shutdown(); + for (const runner of runners) { + await runner.shutdown(); + } + if (usePersistence) { + await IndexedDbTestRunner.destroyPersistence(); + } } } @@ -1053,6 +1259,9 @@ export async function runSpec( export interface SpecConfig { /** A boolean to enable / disable GC. */ useGarbageCollection: boolean; + + /** The number of active clients for this test run. */ + numClients: number; } /** @@ -1060,6 +1269,8 @@ export interface SpecConfig { * set and optionally expected events in the `expect` field. */ export interface SpecStep { + /** The index of the local client for multi-client spec tests. */ + clientIndex?: number; // PORTING NOTE: Only used by web multi-tab tests /** Listen to a new query (must be unique) */ userListen?: SpecUserListen; /** Unlisten from a query (must be listened to) */ @@ -1099,9 +1310,17 @@ export interface SpecStep { */ runTimer?: string; + /** + * Process all events currently enqueued in the AsyncQueue. + */ + drainQueue?: true; + /** Enable or disable RemoteStore's network connection. */ enableNetwork?: boolean; + /** Changes the metadata state of a client instance. */ + applyClientState?: SpecClientState; // PORTING NOTE: Only used by web multi-tab tests + /** Change to a new active user (specified by uid or null for anonymous). */ changeUser?: string | null; @@ -1110,8 +1329,10 @@ export interface SpecStep { * components. This allows you to queue writes, get documents into cache, * etc. and then simulate an app restart. */ - restart?: boolean; + restart?: true; + /** Shut down the client and close it network connection. */ + shutdown?: true; /** * Optional list of expected events. * If not provided, the test will fail if the step causes events to be raised. @@ -1172,15 +1393,28 @@ export type SpecWatchStreamClose = { export type SpecWriteAck = { /** The version the backend uses to ack the write. */ version: TestSnapshotVersion; - /** Whether the ack is expected to generate a user callback. */ - expectUserCallback: boolean; + /** + * Whether we should keep the write in our internal queue. This should only + * be set to 'true' if the client ignores the write (e.g. a secondary client + * which ignores write acknowledgments). + * + * Defaults to false. + */ + // PORTING NOTE: Multi-Tab only. + keepInQueue?: boolean; }; export type SpecWriteFailure = { /** The error the backend uses to fail the write. */ error: SpecError; - /** Whether the failure is expected to generate a user callback. */ - expectUserCallback: boolean; + /** + * Whether we should keep the write in our internal queue. This should be set + * to 'true' for transient errors or if the client ignores the failure + * (e.g. a secondary client which ignores write rejections). + * + * Defaults to false. + */ + keepInQueue?: boolean; }; export interface SpecWatchEntity { @@ -1196,6 +1430,14 @@ export interface SpecWatchEntity { removedTargets?: TargetId[]; } +// PORTING NOTE: Only used by web multi-tab tests. +export type SpecClientState = { + /** The visibility state of the browser tab running the client. */ + visibility?: VisibilityState; + /** Whether this tab should try to forcefully become primary. */ + primary?: true; +}; + /** * [[, ...], , ...] * Note that the last parameter is really of type ...string (spread operator) @@ -1254,16 +1496,45 @@ export interface SpecExpectation { export interface StateExpectation { /** Number of outstanding writes in the datastore queue. */ numOutstandingWrites?: number; + /** Number of clients currently marked active. Used in multi-client tests. */ + numActiveClients?: number; /** Number of requests sent to the write stream. */ writeStreamRequestCount?: number; /** Number of requests sent to the watch stream. */ watchStreamRequestCount?: number; /** Current documents in limbo. Verified in each step until overwritten. */ limboDocs?: string[]; + /** + * Whether the instance holds the primary lease. Used in multi-client tests. + */ + isPrimary?: boolean; /** * Current expected active targets. Verified in each step until overwritten. */ activeTargets?: { [targetId: number]: { query: SpecQuery; resumeToken: string }; }; + /** + * Expected set of callbacks for previously written docs. + */ + userCallbacks?: { + acknowledgedDocs: string[]; + rejectedDocs: string[]; + }; +} + +async function writeOwnerToIndexedDb(clientId: ClientId): Promise { + const db = await SimpleDb.openOrCreate( + IndexedDbTestRunner.TEST_DB_NAME + IndexedDbPersistence.MAIN_DATABASE, + SCHEMA_VERSION, + createOrUpgradeDb + ); + await db.runTransaction('readwrite', ['owner'], txn => { + const owner = txn.store(DbOwner.store); + return owner.put( + 'owner', + new DbOwner(clientId, /* allowTabSynchronization=*/ true, Date.now()) + ); + }); + db.close(); } diff --git a/packages/firestore/test/unit/specs/write_spec.test.ts b/packages/firestore/test/unit/specs/write_spec.test.ts index 26dc03bbff6..a50799c8c79 100644 --- a/packages/firestore/test/unit/specs/write_spec.test.ts +++ b/packages/firestore/test/unit/specs/write_spec.test.ts @@ -20,8 +20,9 @@ import { Code } from '../../../src/util/error'; import { doc, path } from '../../util/helpers'; import { describeSpec, specTest } from './describe_spec'; -import { spec } from './spec_builder'; +import { client, spec } from './spec_builder'; import { RpcError } from './spec_rpc_error'; +import { TimerId } from '../../../src/util/async_queue'; describeSpec('Writes:', [], () => { specTest( @@ -57,7 +58,7 @@ describeSpec('Writes:', [], () => { }) .watchSends({ affects: [query] }, docAv2) .watchSnapshots(2000) - .writeAcks(2000) + .writeAcks('collection/a', 2000) .expectEvents(query, { metadata: [docAv2] }) @@ -68,7 +69,7 @@ describeSpec('Writes:', [], () => { }) .watchSends({ affects: [query] }, docBv2) .watchSnapshots(3000) - .writeAcks(3000) + .writeAcks('collection/b', 3000) .expectEvents(query, { metadata: [docBv2] }); @@ -101,7 +102,7 @@ describeSpec('Writes:', [], () => { }) .watchSends({ affects: [query1] }, doc1c) .watchSnapshots(2000) - .writeAcks(2000) + .writeAcks('collection/key', 2000) .expectEvents(query1, { metadata: [doc1c] }); @@ -136,7 +137,7 @@ describeSpec('Writes:', [], () => { }) .watchSends({ affects: [query1] }, doc1c) .watchSnapshots(watchVersion) - .writeAcks(ackedVersion) // The ack is already outdated by the newer doc1c + .writeAcks('collection/key', ackedVersion) // The ack is already outdated by the newer doc1c .expectEvents(query1, { modified: [doc1c] }); @@ -167,7 +168,7 @@ describeSpec('Writes:', [], () => { modified: [docV2Local] }) // The ack arrives before the watch snapshot; no events yet - .writeAcks(2000) + .writeAcks('collection/key', 2000) .watchSends({ affects: [query] }, docV2) .watchSnapshots(2000) .expectEvents(query, { @@ -200,7 +201,7 @@ describeSpec('Writes:', [], () => { modified: [docV3Local] }) // The ack arrives before the watch snapshot; no events yet - .writeAcks(3000) + .writeAcks('collection/key', 3000) // watch sends some stale data; no events .watchSends({ affects: [query] }, docV2) .watchSnapshots(2000) @@ -251,7 +252,7 @@ describeSpec('Writes:', [], () => { specification.expectNumOutstandingWrites(10); for (let i = 0; i < numWrites; i++) { specification - .writeAcks((i + 1) * 1000) + .writeAcks('collection/a' + i, (i + 1) * 1000) .watchSends({ affects: [query] }, docs[i]) .watchSnapshots((i + 1) * 1000) .expectEvents(query, { @@ -290,7 +291,10 @@ describeSpec('Writes:', [], () => { specification.expectNumOutstandingWrites(10); for (let i = 0; i < numWrites; i++) { specification - .failWrite(new RpcError(Code.PERMISSION_DENIED, 'permission denied')) + .failWrite( + 'collection/a' + i, + new RpcError(Code.PERMISSION_DENIED, 'permission denied') + ) .expectEvents(query, { fromCache: true, hasPendingWrites: i < numWrites - 1, @@ -330,14 +334,17 @@ describeSpec('Writes:', [], () => { .userSets('collection/b', { v: 1 }) .expectEvents(query, { hasPendingWrites: true, added: [docBLocal] }) // ack write but no watch snapshot so it'll be held. - .writeAcks(2000) + .writeAcks('collection/b', 2000) .userSets('collection/a', { v: 2 }) .expectEvents(query, { hasPendingWrites: true, modified: [docAv2Local] }) // reject write, should be released immediately. - .failWrite(new RpcError(Code.PERMISSION_DENIED, 'failure')) + .failWrite( + 'collection/a', + new RpcError(Code.PERMISSION_DENIED, 'failure') + ) .expectEvents(query, { hasPendingWrites: true, modified: [docAv1] }) // watch updates, B should be visible .watchSends({ affects: [query] }, docB) @@ -375,7 +382,7 @@ describeSpec('Writes:', [], () => { added: [docALocal] }) // ack write but without a watch event. - .writeAcks(1000) + .writeAcks('collection/a', 1000) // Do another write. .userSets('collection/b', { v: 1 }) .expectEvents(query, { @@ -383,7 +390,7 @@ describeSpec('Writes:', [], () => { added: [docBLocal] }) // ack second write - .writeAcks(2000) + .writeAcks('collection/b', 2000) // Finally watcher catches up. .watchSends({ affects: [query] }, docA, docB) .watchSnapshots(2000) @@ -417,7 +424,7 @@ describeSpec('Writes:', [], () => { added: [docALocal] }) // ack write but without a watch event. - .writeAcks(1000) + .writeAcks('collection/a', 1000) // handshake + write = 2 requests .expectWriteStreamRequestCount(2) @@ -470,7 +477,7 @@ describeSpec('Writes:', [], () => { added: [docALocal] }) // ack write but without a watch event. - .writeAcks(1000) + .writeAcks('collection/a', 1000) // Unlisten before the write is released. .userUnlistens(query) // Re-add listen and make sure we don't get any events. @@ -507,7 +514,7 @@ describeSpec('Writes:', [], () => { hasPendingWrites: true, added: [doc1a] }) - .failWrite(new RpcError(code, 'failure')) + .failWrite('collection/key', new RpcError(code, 'failure')) .expectEvents(query1, { fromCache: true, removed: [doc1a] @@ -538,9 +545,13 @@ describeSpec('Writes:', [], () => { hasPendingWrites: true, added: [doc1a] }) - .failWrite(new RpcError(Code.RESOURCE_EXHAUSTED, 'transient error'), { - expectUserCallback: false - }); + .failWrite( + 'collection/key', + new RpcError(Code.RESOURCE_EXHAUSTED, 'transient error'), + { + expectUserCallback: false + } + ); } ); @@ -570,10 +581,10 @@ describeSpec('Writes:', [], () => { hasPendingWrites: true, added: [doc1a] }) - .failWrite(new RpcError(code, 'transient error'), { + .failWrite('collection/key', new RpcError(code, 'transient error'), { expectUserCallback: false }) - .writeAcks(1000) + .writeAcks('collection/key', 1000) .watchAcks(query1) .watchSends({ affects: [query1] }, doc1b) .watchCurrents(query1, 'resume-token-1000') @@ -612,7 +623,7 @@ describeSpec('Writes:', [], () => { }) .watchSends({ affects: [query] }, docV2) .watchSnapshots(2000) - .writeAcks(2000) + .writeAcks('collection/doc', 2000) .expectEvents(query, { metadata: [docV2] }) ); } @@ -634,7 +645,630 @@ describeSpec('Writes:', [], () => { expectRequestCount({ handshakes: 2, writes: 2, closes: 1 }) ) .expectNumOutstandingWrites(1) - .writeAcks(1, { expectUserCallback: false }) + .writeAcks('collection/key', 1) .expectNumOutstandingWrites(0); }); + + specTest('New writes are sent after write failure', [], () => { + return spec() + .userSets('collection/a', { v: 1 }) + .failWrite( + 'collection/a', + new RpcError(Code.FAILED_PRECONDITION, 'failure') + ) + .userSets('collection/b', { v: 1 }) + .writeAcks('collection/b', 2000); + }); + + specTest('Primary client acknowledges write', ['multi-client'], () => { + return client(0) + .becomeVisible() + .client(1) + .userSets('collection/a', { v: 1 }) + .client(0) + .writeAcks('collection/a', 1000, { expectUserCallback: false }) + .client(1) + .expectUserCallbacks({ + acknowledged: ['collection/a'] + }); + }); + + specTest('Primary client rejects write', ['multi-client'], () => { + return client(0) + .becomeVisible() + .client(1) + .userSets('collection/a', { v: 1 }) + .client(0) + .failWrite( + 'collection/a', + new RpcError(Code.FAILED_PRECONDITION, 'failure'), + { + expectUserCallback: false + } + ) + .client(1) + .expectUserCallbacks({ + rejected: ['collection/a'] + }); + }); + + specTest( + 'Pending writes are shared between clients', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docV1 = doc( + 'collection/a', + 0, + { v: 1 }, + { hasLocalMutations: true } + ); + const docV2 = doc( + 'collection/a', + 0, + { v: 2 }, + { hasLocalMutations: true } + ); + const docV3 = doc( + 'collection/a', + 0, + { v: 3 }, + { hasLocalMutations: true } + ); + + return client(0) + .userListens(query) + .watchAcksFull(query, 500) + .expectEvents(query, {}) + .userSets('collection/a', { v: 1 }) + .expectEvents(query, { + hasPendingWrites: true, + added: [docV1] + }) + .client(1) + .userListens(query) + .expectEvents(query, { + hasPendingWrites: true, + added: [docV1] + }) + .client(0) + .userSets('collection/a', { v: 2 }) + .expectEvents(query, { + hasPendingWrites: true, + modified: [docV2] + }) + .client(1) + .expectEvents(query, { + hasPendingWrites: true, + modified: [docV2] + }) + .userSets('collection/a', { v: 3 }) + .expectEvents(query, { + hasPendingWrites: true, + modified: [docV3] + }) + .client(0) + .expectEvents(query, { + hasPendingWrites: true, + modified: [docV3] + }); + } + ); + + specTest( + 'Pending write is acknowledged by primary client', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const localDoc = doc( + 'collection/a', + 0, + { v: 1 }, + { hasLocalMutations: true } + ); + const remoteDoc = doc('collection/a', 1000, { v: 1 }); + return client(0) + .becomeVisible() + .userListens(query) + .watchAcksFull(query, 500) + .expectEvents(query, {}) + .client(1) + .userListens(query) + .expectEvents(query, {}) + .userSets('collection/a', { v: 1 }) + .expectEvents(query, { + hasPendingWrites: true, + added: [localDoc] + }) + .client(0) + .expectEvents(query, { + hasPendingWrites: true, + added: [localDoc] + }) + .writeAcks('collection/a', 1000, { expectUserCallback: false }) + .watchSends({ affects: [query] }, remoteDoc) + .watchSnapshots(1000) + .expectEvents(query, { + metadata: [remoteDoc] + }) + .client(1) + .expectUserCallbacks({ + acknowledged: ['collection/a'] + }) + .expectEvents(query, { + metadata: [remoteDoc] + }); + } + ); + + specTest( + 'Pending write is rejected by primary client', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const localDoc = doc( + 'collection/a', + 0, + { v: 1 }, + { hasLocalMutations: true } + ); + + return client(0) + .userListens(query) + .watchAcksFull(query, 500) + .expectEvents(query, {}) + .client(1) + .userListens(query) + .expectEvents(query, {}) + .userSets('collection/a', { v: 1 }) + .expectEvents(query, { + hasPendingWrites: true, + added: [localDoc] + }) + .client(0) + .expectEvents(query, { + hasPendingWrites: true, + added: [localDoc] + }) + .failWrite( + 'collection/a', + new RpcError(Code.FAILED_PRECONDITION, 'failure'), + { + expectUserCallback: false + } + ) + .expectEvents(query, { + removed: [localDoc] + }) + .client(1) + .expectUserCallbacks({ + rejected: ['collection/a'] + }) + .expectEvents(query, { + removed: [localDoc] + }); + } + ); + + specTest('Held write is released by primary client', ['multi-client'], () => { + const query = Query.atPath(path('collection')); + const docALocal = doc( + 'collection/a', + 0, + { v: 1 }, + { hasLocalMutations: true } + ); + const docA = doc('collection/a', 1000, { v: 1 }); + + return ( + client(0) + .userListens(query) + .watchAcksFull(query, 500) + .expectEvents(query, {}) + .client(1) + .userSets('collection/a', { v: 1 }) + .client(0) + .expectEvents(query, { + hasPendingWrites: true, + added: [docALocal] + }) + // Ack write but without a watch event. + .writeAcks('collection/a', 1000, { expectUserCallback: false }) + .client(1) // No events + .client(0) + // Watcher catches up. + .watchSends({ affects: [query] }, docA) + .watchSnapshots(2000) + .expectEvents(query, { + metadata: [docA] + }) + .client(1) + .expectUserCallbacks({ + acknowledged: ['collection/a'] + }) + ); + }); + + specTest('Write are sequenced by multiple clients', ['multi-client'], () => { + return client(0) + .userSets('collection/a', { v: 1 }) + .client(1) + .userSets('collection/b', { v: 1 }) + .client(2) + .userSets('collection/c', { v: 1 }) + .client(3) + .userSets('collection/d', { v: 1 }) + .client(0) + .writeAcks('collection/a', 1000) + .writeAcks('collection/b', 2000, { expectUserCallback: false }) + .writeAcks('collection/c', 3000, { expectUserCallback: false }) + .failWrite( + 'collection/d', + new RpcError(Code.FAILED_PRECONDITION, 'failure'), + { + expectUserCallback: false + } + ) + .client(1) + .expectUserCallbacks({ + acknowledged: ['collection/b'] + }) + .client(2) + .expectUserCallbacks({ + acknowledged: ['collection/c'] + }) + .client(3) + .expectUserCallbacks({ + rejected: ['collection/d'] + }) + .client(0) + .userSets('collection/f', { v: 1 }) + .client(1) + .userSets('collection/g', { v: 1 }) + .client(2) + .userSets('collection/h', { v: 1 }) + .client(3) + .userSets('collection/i', { v: 1 }) + .client(0) + .writeAcks('collection/f', 4000) + .writeAcks('collection/g', 5000, { expectUserCallback: false }) + .writeAcks('collection/h', 6000, { expectUserCallback: false }) + .failWrite( + 'collection/i', + new RpcError(Code.FAILED_PRECONDITION, 'failure'), + { + expectUserCallback: false + } + ) + .client(1) + .expectUserCallbacks({ + acknowledged: ['collection/g'] + }) + .client(2) + .expectUserCallbacks({ + acknowledged: ['collection/h'] + }) + .client(3) + .expectUserCallbacks({ + rejected: ['collection/i'] + }) + .client(3) + .userSets('collection/j', { v: 1 }) + .userSets('collection/k', { v: 1 }) + .userSets('collection/l', { v: 1 }) + .client(0) + .writeAcks('collection/j', 7000, { expectUserCallback: false }) + .failWrite( + 'collection/k', + new RpcError(Code.FAILED_PRECONDITION, 'failure'), + { + expectUserCallback: false + } + ) + .writeAcks('collection/k', 8000, { expectUserCallback: false }) + .client(3) + .expectUserCallbacks({ + acknowledged: ['collection/j', 'collection/l'], + rejected: ['collection/k'] + }); + }); + + specTest( + 'Write is executed after primary tab failover', + ['multi-client'], + () => { + return client(0) + .becomeVisible() + .expectPrimaryState(true) + .client(1) + .expectPrimaryState(false) + .userSets('collection/a', { v: 1 }) + .userSets('collection/b', { v: 1 }) + .client(0) + .writeAcks('collection/a', 1000, { expectUserCallback: false }) + .shutdown() + .client(1) + .expectUserCallbacks({ + acknowledged: ['collection/a'] + }) + .runTimer(TimerId.ClientMetadataRefresh) + .expectPrimaryState(true) + .writeAcks('collection/b', 2000); + } + ); + + specTest('Secondary tabs handle user change', ['multi-client'], () => { + const query = Query.atPath(path('collection')); + const docALocal = doc( + 'collection/a', + 0, + { v: 1 }, + { hasLocalMutations: true } + ); + const docBLocal = doc( + 'collection/b', + 0, + { v: 1 }, + { hasLocalMutations: true } + ); + const docCLocal = doc( + 'collection/c', + 0, + { v: 1 }, + { hasLocalMutations: true } + ); + + // Firebase Auth attempts to rapidly synchronize user changes across tabs. + // We emulate this behavior in this spec tests by calling `changeUser` + // manually for all clients. + return ( + client(0) + .userListens(query) + .changeUser('user1') + // User 1 writes `docA` + .userSets('collection/a', { v: 1 }) + .expectEvents(query, { + added: [docALocal], + fromCache: true, + hasPendingWrites: true + }) + .client(1) + .changeUser('user1') + .userListens(query) + .expectEvents(query, { + added: [docALocal], + fromCache: true, + hasPendingWrites: true + }) + // User 1 sets `docB` from a different tab + .userSets('collection/b', { v: 1 }) + .expectEvents(query, { + added: [docBLocal], + fromCache: true, + hasPendingWrites: true + }) + .client(0) + .expectEvents(query, { + added: [docBLocal], + fromCache: true, + hasPendingWrites: true + }) + .changeUser('user2') + .expectEvents(query, { + removed: [docALocal, docBLocal], + fromCache: true + }) + // User 2 adds `docC` + .userSets('collection/c', { v: 1 }) + .expectEvents(query, { + added: [docCLocal], + fromCache: true, + hasPendingWrites: true + }) + .client(1) + .changeUser('user2') + .expectEvents(query, { + removed: [docALocal, docBLocal], + added: [docCLocal], + fromCache: true, + hasPendingWrites: true + }) + .changeUser('user1') + .expectEvents(query, { + added: [docALocal, docBLocal], + removed: [docCLocal], + fromCache: true, + hasPendingWrites: true + }) + .client(0) + .changeUser('user1') + .expectEvents(query, { + added: [docALocal, docBLocal], + removed: [docCLocal], + fromCache: true, + hasPendingWrites: true + }) + ); + }); + + specTest('Mutations are scoped by user', ['multi-client'], () => { + const query = Query.atPath(path('collection')); + const docALocal = doc( + 'collection/a', + 0, + { v: 1 }, + { hasLocalMutations: true } + ); + const docBLocal = doc( + 'collection/b', + 0, + { v: 1 }, + { hasLocalMutations: true } + ); + + return client(0) + .changeUser('user1') + .userSets('collection/a', { v: 1 }) + .client(1) + .changeUser('user2') + .userSets('collection/b', { v: 1 }) + .client(0) + .userListens(query) + .expectEvents(query, { + added: [docALocal], + fromCache: true, + hasPendingWrites: true + }) + .client(1) + .userListens(query) + .expectEvents(query, { + added: [docBLocal], + fromCache: true, + hasPendingWrites: true + }); + }); + + specTest('Mutation recovers after primary takeover', ['multi-client'], () => { + const query = Query.atPath(path('collection')); + const docALocal = doc( + 'collection/a', + 0, + { k: 'a' }, + { hasLocalMutations: true } + ); + const docA = doc('collection/a', 1000, { k: 'a' }); + + return client(0) + .expectPrimaryState(true) + .userSets('collection/a', { k: 'a' }) + .client(1) + .userListens(query) + .expectEvents(query, { + added: [docALocal], + hasPendingWrites: true, + fromCache: true + }) + .stealPrimaryLease() + .writeAcks('collection/a', 1000, { expectUserCallback: false }) + .watchAcksFull(query, 1000, docA) + .expectEvents(query, { metadata: [docA] }) + .client(0) + .expectUserCallbacks({ + acknowledged: ['collection/a'] + }); + }); + + specTest('Write is sent by newly started primary', ['multi-client'], () => { + return client(0) + .expectPrimaryState(true) + .client(1) + .expectPrimaryState(false) + .userSets('collection/a', { v: 1 }) + .client(0) + .shutdown() + .client(2) + .expectPrimaryState(true) + .expectNumOutstandingWrites(1) + .writeAcks('collection/a', 1000, { expectUserCallback: false }) + .client(1) + .expectUserCallbacks({ + acknowledged: ['collection/a'] + }); + }); + + specTest( + 'Unresponsive primary ignores acknowledged write', + ['multi-client'], + () => { + return ( + client(0) + .expectPrimaryState(true) + // Send initial write to open the write stream + .userSets('collection/a', { k: 'a' }) + .writeAcks('collection/a', 1000) + .client(1) + .userSets('collection/b', { k: 'b' }) + .client(2) + .stealPrimaryLease() + .client(0) + // Client 2 is now the primary client, and client 0 ignores the write + // acknowledgement. + .writeAcks('collection/b', 2000, { + expectUserCallback: false, + keepInQueue: true + }) + .client(2) + .writeAcks('collection/b', 2000, { expectUserCallback: false }) + .client(1) + .expectUserCallbacks({ + acknowledged: ['collection/b'] + }) + ); + } + ); + + specTest( + 'Unresponsive primary ignores rejected write', + ['multi-client'], + () => { + return ( + client(0) + .expectPrimaryState(true) + // Send initial write to open the write stream + .userSets('collection/a', { k: 'a' }) + .writeAcks('collection/a', 1000) + .client(1) + .userSets('collection/b', { k: 'b' }) + .client(2) + .stealPrimaryLease() + .client(0) + // Client 2 is now the primary client, and client 0 ignores the rejected + // write. + .failWrite( + 'collection/b', + new RpcError(Code.FAILED_PRECONDITION, 'Write error'), + { + expectUserCallback: false, + keepInQueue: true + } + ) + .client(2) + .failWrite( + 'collection/b', + new RpcError(Code.FAILED_PRECONDITION, 'Write error'), + { expectUserCallback: false } + ) + .client(1) + .expectUserCallbacks({ + rejected: ['collection/b'] + }) + ); + } + ); + + specTest( + 'Mutation are not sent twice after primary failover', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 0, { k: 'a' }); + const docB = doc('collection/b', 0, { k: 'b' }); + + return client(0) + .expectPrimaryState(true) + .userSets('collection/a', { k: 'a' }) + .userSets('collection/b', { k: 'b' }) + .client(1) + .stealPrimaryLease() + .writeAcks('collection/a', 1000, { expectUserCallback: false }) + .client(0) + .expectUserCallbacks({ + acknowledged: ['collection/a'] + }) + .stealPrimaryLease() + .writeAcks('collection/b', 2000) + .userListens(query) + .expectEvents(query, { added: [docA, docB], fromCache: true }); + } + ); }); diff --git a/packages/firestore/test/unit/util/misc.test.ts b/packages/firestore/test/unit/util/misc.test.ts index f215d3d25f8..0fe8461738d 100644 --- a/packages/firestore/test/unit/util/misc.test.ts +++ b/packages/firestore/test/unit/util/misc.test.ts @@ -15,25 +15,7 @@ */ import { expect } from 'chai'; -import { - immediatePredecessor, - immediateSuccessor -} from '../../../src/util/misc'; - -describe('immediatePredecessor', () => { - it('generates the correct immediate predecessor', () => { - expect(immediatePredecessor('b')).to.equal('a'); - expect(immediatePredecessor('bbBB')).to.equal('bbBA'); - expect(immediatePredecessor('aaa\0')).to.equal('aaa'); - expect(immediatePredecessor('\0')).to.equal(''); - expect(immediatePredecessor('\0\0\0')).to.equal('\0\0'); - expect(immediatePredecessor('az\u00e0')).to.equal('az\u00df'); - expect(immediatePredecessor('\uffff\uffff\uffff')).to.equal( - '\uffff\uffff\ufffe' - ); - expect(immediatePredecessor('')).to.equal(''); - }); -}); +import { immediateSuccessor } from '../../../src/util/misc'; describe('immediateSuccessor', () => { it('generates the correct immediate successors', () => { diff --git a/packages/firestore/test/util/helpers.ts b/packages/firestore/test/util/helpers.ts index 7bb516c80ea..7dd405d6825 100644 --- a/packages/firestore/test/util/helpers.ts +++ b/packages/firestore/test/util/helpers.ts @@ -127,6 +127,10 @@ export function deletedDoc( return new NoDocument(key(keyStr), version(ver)); } +export function removedDoc(keyStr: string): NoDocument { + return new NoDocument(key(keyStr), SnapshotVersion.forDeletedDoc()); +} + export function wrap(value: AnyJs): FieldValue { // HACK: We use parseQueryValue() since it accepts scalars as well as // arrays / objects, and our tests currently use wrap() pretty generically so @@ -457,7 +461,7 @@ export function applyDocChanges( ...docsOrKeys: Array ): ViewChange { const changes = view.computeDocChanges(documentUpdates(...docsOrKeys)); - return view.applyChanges(changes); + return view.applyChanges(changes, true); } /** diff --git a/packages/firestore/test/util/node_persistence.ts b/packages/firestore/test/util/node_persistence.ts index ac91b597d86..ab23bce2b3d 100644 --- a/packages/firestore/test/util/node_persistence.ts +++ b/packages/firestore/test/util/node_persistence.ts @@ -18,6 +18,8 @@ import * as registerIndexedDBShim from 'indexeddbshim'; import * as fs from 'fs'; import * as os from 'os'; +import { FakeWindow, SharedFakeWebStorage } from './test_platform'; + // WARNING: The `indexeddbshim` installed via this module should only ever be // used during initial development. Always validate your changes via // `yarn test:browser` (which uses a browser-based IndexedDB implementation) @@ -36,7 +38,19 @@ if (process.env.USE_MOCK_PERSISTENCE === 'YES') { databaseBasePath: dbDir, deleteDatabaseFiles: true }); - globalAny.window = Object.assign(globalAny.window || {}, { - indexedDB: globalAny.indexedDB - }); + + const fakeWindow = new FakeWindow( + new SharedFakeWebStorage(), + globalAny.indexedDB + ); + + globalAny.window = fakeWindow; + + // We need to define the `Event` type as it is used in Node to send events to + // WebStorage when using both the IndexedDB mock and the WebStorage mock. + class Event { + constructor(typeArg: string, eventInitDict?: EventInit) {} + } + + globalAny.Event = Event; } diff --git a/packages/firestore/test/util/test_platform.ts b/packages/firestore/test/util/test_platform.ts new file mode 100644 index 00000000000..aba01e0cd05 --- /dev/null +++ b/packages/firestore/test/util/test_platform.ts @@ -0,0 +1,262 @@ +/** + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DatabaseId, DatabaseInfo } from '../../src/core/database_info'; +import { AnyJs } from '../../src/util/misc'; +import { Platform } from '../../src/platform/platform'; +import { JsonProtoSerializer } from '../../src/remote/serializer'; +import { ProtoByteString } from '../../src/core/types'; +import { Connection } from '../../src/remote/connection'; +import { assert, fail } from '../../src/util/assert'; + +/** + * `Window` fake that implements the event and storage API that is used by + * Firestore. + */ +export class FakeWindow { + private readonly fakeStorageArea: Storage; + private readonly fakeIndexedDb: IDBFactory; + + private storageListeners: EventListener[] = []; + + constructor( + sharedFakeStorage: SharedFakeWebStorage, + fakeIndexedDb?: IDBFactory + ) { + this.fakeStorageArea = sharedFakeStorage.getStorageArea(event => { + for (const listener of this.storageListeners) { + listener(event); + } + }); + this.fakeIndexedDb = + fakeIndexedDb || (typeof window !== 'undefined' && window.indexedDB); + } + + get localStorage(): Storage { + return this.fakeStorageArea; + } + + get indexedDB(): IDBFactory { + return this.fakeIndexedDb; + } + + addEventListener(type: string, listener: EventListener): void { + switch (type) { + case 'storage': + this.storageListeners.push(listener); + break; + case 'unload': + // The spec tests currently do not rely on 'unload' listeners. + break; + default: + fail(`MockWindow doesn't support events of type '${type}'`); + } + } + + removeEventListener(type: string, listener: EventListener): void { + if (type === 'storage') { + const oldCount = this.storageListeners.length; + this.storageListeners = this.storageListeners.filter( + registeredListener => listener !== registeredListener + ); + const newCount = this.storageListeners.length; + assert( + newCount === oldCount - 1, + "Listener passed to 'removeEventListener' doesn't match any registered listener." + ); + } + } +} + +/** + * `Document` fake that implements the `visibilitychange` API used by Firestore. + */ +export class FakeDocument { + private _visibilityState: VisibilityState = 'unloaded'; + private visibilityListener: EventListener | null; + + get visibilityState(): VisibilityState { + return this._visibilityState; + } + + addEventListener(type: string, listener: EventListener): void { + assert( + type === 'visibilitychange', + "FakeDocument only supports events of type 'visibilitychange'" + ); + this.visibilityListener = listener; + } + + removeEventListener(type: string, listener: EventListener): void { + if (listener === this.visibilityListener) { + this.visibilityListener = null; + } + } + + raiseVisibilityEvent(visibility: VisibilityState): void { + this._visibilityState = visibility; + if (this.visibilityListener) { + this.visibilityListener(new Event('visibilitychange')); + } + } +} + +/** + * `WebStorage` mock that implements the WebStorage behavior for multiple + * clients. To get a client-specific storage area that implements the WebStorage + * API, invoke `getStorageArea(storageListener)`. + */ +export class SharedFakeWebStorage { + private readonly data = new Map(); + private readonly activeClients: Array<{ + storageListener: EventListener; + storageArea: Storage; + }> = []; + + getStorageArea(storageListener: EventListener): Storage { + const clientIndex = this.activeClients.length; + const self = this; + + const storageArea: Storage = { + get length(): number { + return self.length; + }, + getItem: (key: string) => this.getItem(key), + key: (index: number) => this.key(index), + clear: () => this.clear(), + removeItem: (key: string) => { + const oldValue = this.getItem(key); + this.removeItem(key); + this.raiseStorageEvent(clientIndex, key, oldValue, null); + }, + setItem: (key: string, value: string) => { + const oldValue = this.getItem(key); + this.setItem(key, value); + this.raiseStorageEvent(clientIndex, key, oldValue, value); + } + }; + + this.activeClients[clientIndex] = { storageListener, storageArea }; + + return storageArea; + } + + private clear(): void { + this.data.clear(); + } + + private getItem(key: string): string | null { + return this.data.has(key) ? this.data.get(key) : null; + } + + private key(index: number): string | null { + const key = Array.from(this.data.keys())[index]; + return key !== undefined ? key : null; + } + + private removeItem(key: string): void { + this.data.delete(key); + } + + private setItem(key: string, data: string): void { + this.data.set(key, data); + } + + private get length(): number { + return this.data.size; + } + + private raiseStorageEvent( + sourceClientIndex: number, + key: string, + oldValue: string | null, + newValue: string | null + ): void { + this.activeClients.forEach((client, index) => { + // WebStorage doesn't raise events for writes from the originating client. + if (sourceClientIndex === index) { + return; + } + + client.storageListener({ + key, + oldValue, + newValue, + storageArea: client.storageArea + } as any); // tslint:disable-line:no-any Not mocking entire Event type. + }); + } +} + +/** + * Implementation of `Platform` that allows faking of `document` and `window`. + */ +export class TestPlatform implements Platform { + readonly mockDocument: FakeDocument | null = null; + readonly mockWindow: FakeWindow | null = null; + + constructor( + private readonly basePlatform: Platform, + private readonly mockStorage: SharedFakeWebStorage + ) { + this.mockDocument = new FakeDocument(); + this.mockWindow = new FakeWindow(this.mockStorage); + } + + get document(): Document | null { + // tslint:disable-next-line:no-any FakeWindow doesn't support full Document interface. + return this.mockDocument as any; + } + + get window(): Window | null { + // tslint:disable-next-line:no-any FakeWindow doesn't support full Window interface. + return this.mockWindow as any; + } + + get base64Available(): boolean { + return this.basePlatform.base64Available; + } + + get emptyByteString(): ProtoByteString { + return this.basePlatform.emptyByteString; + } + + raiseVisibilityEvent(visibility: VisibilityState): void { + if (this.mockDocument) { + this.mockDocument.raiseVisibilityEvent(visibility); + } + } + + loadConnection(databaseInfo: DatabaseInfo): Promise { + return this.basePlatform.loadConnection(databaseInfo); + } + + newSerializer(databaseId: DatabaseId): JsonProtoSerializer { + return this.basePlatform.newSerializer(databaseId); + } + + formatJSON(value: AnyJs): string { + return this.basePlatform.formatJSON(value); + } + + atob(encoded: string): string { + return this.basePlatform.atob(encoded); + } + + btoa(raw: string): string { + return this.basePlatform.btoa(raw); + } +}