diff --git a/.changeset/lemon-steaks-draw.md b/.changeset/lemon-steaks-draw.md new file mode 100644 index 00000000000..f920c46404e --- /dev/null +++ b/.changeset/lemon-steaks-draw.md @@ -0,0 +1,5 @@ +--- +"@firebase/firestore": internal +--- + +Merge bundle loading implementation without exposing public API diff --git a/packages-exp/auth-exp/index.webworker.ts b/packages-exp/auth-exp/index.webworker.ts index 886105720b8..e9b29a71912 100644 --- a/packages-exp/auth-exp/index.webworker.ts +++ b/packages-exp/auth-exp/index.webworker.ts @@ -37,7 +37,10 @@ registerAuth(ClientPlatform.WORKER); export function getAuth(app = getApp()): Auth { // Unlike the other environments, we need to explicitly check if indexedDb is // available. That means doing the whole rigamarole - const auth = _getProvider(app, _ComponentName.AUTH).getImmediate() as AuthImpl; + const auth = _getProvider( + app, + _ComponentName.AUTH + ).getImmediate() as AuthImpl; // This promise is intended to float; auth initialization happens in the // background, meanwhile the auth object may be used by the app. diff --git a/packages/database/src/api/Database.ts b/packages/database/src/api/Database.ts index b75e09cdced..94f321cc297 100644 --- a/packages/database/src/api/Database.ts +++ b/packages/database/src/api/Database.ts @@ -157,14 +157,17 @@ export class Database implements FirebaseService { validateUrl(apiName, 1, parsedURL); const repoInfo = parsedURL.repoInfo; - if (!repoInfo.isCustomHost() && repoInfo.host !== this.repo_.repoInfo_.host) { + if ( + !repoInfo.isCustomHost() && + repoInfo.host !== this.repo_.repoInfo_.host + ) { fatal( apiName + ': Host name does not match the current database: ' + '(found ' + repoInfo.host + ' but expected ' + - this.repo_.repoInfo_.host+ + this.repo_.repoInfo_.host + ')' ); } diff --git a/packages/firestore/exp/index.ts b/packages/firestore/exp/index.ts index 5dcb193266c..5ac7079c8ee 100644 --- a/packages/firestore/exp/index.ts +++ b/packages/firestore/exp/index.ts @@ -31,8 +31,7 @@ export { waitForPendingWrites, disableNetwork, enableNetwork, - terminate, - Settings + terminate } from './src/api/database'; export { diff --git a/packages/firestore/externs.json b/packages/firestore/externs.json index 74f5d2a30bb..06e8519f3c6 100644 --- a/packages/firestore/externs.json +++ b/packages/firestore/externs.json @@ -28,6 +28,7 @@ "packages/webchannel-wrapper/src/index.d.ts", "packages/util/dist/src/crypt.d.ts", "packages/util/dist/src/environment.d.ts", + "packages/firestore/src/protos/firestore_bundle_proto.ts", "packages/firestore/src/protos/firestore_proto_api.d.ts", "packages/firestore/src/util/error.ts", "packages/firestore/src/local/indexeddb_schema.ts", diff --git a/packages/firestore/rollup.shared.js b/packages/firestore/rollup.shared.js index d748bfd0d73..009facb4041 100644 --- a/packages/firestore/rollup.shared.js +++ b/packages/firestore/rollup.shared.js @@ -129,7 +129,75 @@ const manglePrivatePropertiesOptions = { }, mangle: { properties: { - regex: /^__PRIVATE_/ + regex: /^__PRIVATE_/, + // All JS Keywords are reserved. Although this should be taken cared of by + // Terser, we have seen issues with `do`, hence the extra caution. + reserved: [ + 'abstract', + 'arguments', + 'await', + 'boolean', + 'break', + 'byte', + 'case', + 'catch', + 'char', + 'class', + 'const', + 'continue', + 'debugger', + 'default', + 'delete', + 'do', + 'double', + 'else', + 'enum', + 'eval', + 'export', + 'extends', + 'false', + 'final', + 'finally', + 'float', + 'for', + 'function', + 'goto', + 'if', + 'implements', + 'import', + 'in', + 'instanceof', + 'int', + 'interface', + 'let', + 'long', + 'native', + 'new', + 'null', + 'package', + 'private', + 'protected', + 'public', + 'return', + 'short', + 'static', + 'super', + 'switch', + 'synchronized', + 'this', + 'throw', + 'throws', + 'transient', + 'true', + 'try', + 'typeof', + 'var', + 'void', + 'volatile', + 'while', + 'with', + 'yield' + ] } } }; diff --git a/packages/firestore/src/api/bundle.ts b/packages/firestore/src/api/bundle.ts new file mode 100644 index 00000000000..8bd3b4b74fa --- /dev/null +++ b/packages/firestore/src/api/bundle.ts @@ -0,0 +1,140 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Deferred } from '../util/promise'; +import { PartialObserver } from './observer'; +import { debugAssert } from '../util/assert'; +import { FirestoreError } from '../util/error'; + +export interface ApiLoadBundleTask { + onProgress( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + next?: (progress: ApiLoadBundleTaskProgress) => any, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + error?: (error: Error) => any, + complete?: () => void + ): void; + + then( + onFulfilled?: (a: ApiLoadBundleTaskProgress) => T | PromiseLike, + onRejected?: (a: Error) => R | PromiseLike + ): Promise; + + catch( + onRejected: (a: Error) => R | PromiseLike + ): Promise; +} + +export interface ApiLoadBundleTaskProgress { + documentsLoaded: number; + totalDocuments: number; + bytesLoaded: number; + totalBytes: number; + taskState: TaskState; +} + +export type TaskState = 'Error' | 'Running' | 'Success'; + +export class LoadBundleTask + implements ApiLoadBundleTask, PromiseLike { + private _progressObserver: PartialObserver = {}; + private _taskCompletionResolver = new Deferred(); + + private _lastProgress: ApiLoadBundleTaskProgress = { + taskState: 'Running', + totalBytes: 0, + totalDocuments: 0, + bytesLoaded: 0, + documentsLoaded: 0 + }; + + onProgress( + next?: (progress: ApiLoadBundleTaskProgress) => unknown, + error?: (err: Error) => unknown, + complete?: () => void + ): void { + this._progressObserver = { + next, + error, + complete + }; + } + + catch( + onRejected: (a: Error) => R | PromiseLike + ): Promise { + return this._taskCompletionResolver.promise.catch(onRejected); + } + + then( + onFulfilled?: (a: ApiLoadBundleTaskProgress) => T | PromiseLike, + onRejected?: (a: Error) => R | PromiseLike + ): Promise { + return this._taskCompletionResolver.promise.then(onFulfilled, onRejected); + } + + /** + * Notifies all observers that bundle loading has completed, with a provided + * `LoadBundleTaskProgress` object. + */ + _completeWith(progress: ApiLoadBundleTaskProgress): void { + debugAssert( + progress.taskState === 'Success', + 'Task is not completed with Success.' + ); + this._updateProgress(progress); + if (this._progressObserver.complete) { + this._progressObserver.complete(); + } + + this._taskCompletionResolver.resolve(progress); + } + + /** + * Notifies all observers that bundle loading has failed, with a provided + * `Error` as the reason. + */ + _failWith(error: FirestoreError): void { + this._lastProgress.taskState = 'Error'; + + if (this._progressObserver.next) { + this._progressObserver.next(this._lastProgress); + } + + if (this._progressObserver.error) { + this._progressObserver.error(error); + } + + this._taskCompletionResolver.reject(error); + } + + /** + * Notifies a progress update of loading a bundle. + * @param progress The new progress. + */ + _updateProgress(progress: ApiLoadBundleTaskProgress): void { + debugAssert( + this._lastProgress.taskState === 'Running', + 'Cannot update progress on a completed or failed task' + ); + + this._lastProgress = progress; + if (this._progressObserver.next) { + this._progressObserver.next(progress); + } + } +} diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index 7beafb8f861..5001ff8b6af 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -25,7 +25,9 @@ import { FirestoreClient, firestoreClientGetDocumentsFromLocalCache, firestoreClientGetDocumentsViaSnapshotListener, + firestoreClientGetNamedQuery, firestoreClientListen, + firestoreClientLoadBundle, firestoreClientTransaction, firestoreClientWrite } from '../core/firestore_client'; @@ -157,6 +159,7 @@ import { import { makeDatabaseInfo } from '../../lite/src/api/database'; import { DEFAULT_HOST } from '../../lite/src/api/components'; +import { ApiLoadBundleTask, LoadBundleTask } from './bundle'; /** * Constant used to indicate the LRU garbage collection should be disabled. @@ -440,6 +443,36 @@ export function setLogLevel(level: PublicLogLevel): void { setClientLogLevel(level); } +export function loadBundle( + db: Firestore, + bundleData: ArrayBuffer | ReadableStream | string +): ApiLoadBundleTask { + const resultTask = new LoadBundleTask(); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + firestoreClientLoadBundle( + ensureFirestoreConfigured(db._delegate), + bundleData, + resultTask + ); + return resultTask; +} + +export function namedQuery( + db: Firestore, + name: string +): Promise { + return firestoreClientGetNamedQuery( + ensureFirestoreConfigured(db._delegate), + name + ).then(namedQuery => { + if (!namedQuery) { + return null; + } + + return new Query(namedQuery.query, db, null); + }); +} + /** * A reference to a transaction. */ diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts new file mode 100644 index 00000000000..e68c28ed2eb --- /dev/null +++ b/packages/firestore/src/core/bundle.ts @@ -0,0 +1,264 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Query } from './query'; +import { SnapshotVersion } from './snapshot_version'; +import { + fromDocument, + fromName, + fromVersion, + JsonProtoSerializer +} from '../remote/serializer'; +import * as bundleProto from '../protos/firestore_bundle_proto'; +import { BundleMetadata } from '../protos/firestore_bundle_proto'; +import * as api from '../protos/firestore_proto_api'; +import { DocumentKey } from '../model/document_key'; +import { MaybeDocument, NoDocument } from '../model/document'; +import { debugAssert } from '../util/assert'; +import { + applyBundleDocuments, + LocalStore, + saveNamedQuery +} from '../local/local_store'; +import { SizedBundleElement } from '../util/bundle_reader'; +import { + documentKeySet, + DocumentKeySet, + MaybeDocumentMap +} from '../model/collections'; +import { ApiLoadBundleTaskProgress } from '../api/bundle'; + +/** + * Represents a Firestore bundle saved by the SDK in its local storage. + */ +export interface Bundle { + readonly id: string; + readonly version: number; + /** + * Set to the snapshot version of the bundle if created by the Server SDKs. + * Otherwise set to SnapshotVersion.MIN. + */ + readonly createTime: SnapshotVersion; +} + +/** + * Represents a Query saved by the SDK in its local storage. + */ +export interface NamedQuery { + readonly name: string; + readonly query: Query; + /** The time at which the results for this query were read. */ + readonly readTime: SnapshotVersion; +} + +/** + * Represents a bundled document, including the metadata and the document + * itself, if it exists. + */ +interface BundledDocument { + metadata: bundleProto.BundledDocumentMetadata; + document?: api.Document; +} + +/** + * An array of `BundledDocument`. + */ +export type BundledDocuments = BundledDocument[]; + +/** + * Helper to convert objects from bundles to model objects in the SDK. + */ +export class BundleConverter { + constructor(private readonly serializer: JsonProtoSerializer) {} + + toDocumentKey(name: string): DocumentKey { + return fromName(this.serializer, name); + } + + /** + * Converts a BundleDocument to a MaybeDocument. + */ + toMaybeDocument(bundledDoc: BundledDocument): MaybeDocument { + if (bundledDoc.metadata.exists) { + debugAssert( + !!bundledDoc.document, + 'Document is undefined when metadata.exist is true.' + ); + return fromDocument(this.serializer, bundledDoc.document!, false); + } else { + return new NoDocument( + this.toDocumentKey(bundledDoc.metadata.name!), + this.toSnapshotVersion(bundledDoc.metadata.readTime!) + ); + } + } + + toSnapshotVersion(time: api.Timestamp): SnapshotVersion { + return fromVersion(time); + } +} + +/** + * Returns a `LoadBundleTaskProgress` representing the initial progress of + * loading a bundle. + */ +export function bundleInitialProgress( + metadata: BundleMetadata +): ApiLoadBundleTaskProgress { + return { + taskState: 'Running', + documentsLoaded: 0, + bytesLoaded: 0, + totalDocuments: metadata.totalDocuments!, + totalBytes: metadata.totalBytes! + }; +} + +/** + * Returns a `LoadBundleTaskProgress` representing the progress that the loading + * has succeeded. + */ +export function bundleSuccessProgress( + metadata: BundleMetadata +): ApiLoadBundleTaskProgress { + return { + taskState: 'Success', + documentsLoaded: metadata.totalDocuments!, + bytesLoaded: metadata.totalBytes!, + totalDocuments: metadata.totalDocuments!, + totalBytes: metadata.totalBytes! + }; +} + +export class BundleLoadResult { + constructor( + readonly progress: ApiLoadBundleTaskProgress, + readonly changedDocs: MaybeDocumentMap + ) {} +} + +/** + * A class to process the elements from a bundle, load them into local + * storage and provide progress update while loading. + */ +export class BundleLoader { + /** The current progress of loading */ + private progress: ApiLoadBundleTaskProgress; + /** Batched queries to be saved into storage */ + private queries: bundleProto.NamedQuery[] = []; + /** Batched documents to be saved into storage */ + private documents: BundledDocuments = []; + + constructor( + private bundleMetadata: bundleProto.BundleMetadata, + private localStore: LocalStore, + private serializer: JsonProtoSerializer + ) { + this.progress = bundleInitialProgress(bundleMetadata); + } + + /** + * Adds an element from the bundle to the loader. + * + * Returns a new progress if adding the element leads to a new progress, + * otherwise returns null. + */ + addSizedElement( + element: SizedBundleElement + ): ApiLoadBundleTaskProgress | null { + debugAssert(!element.isBundleMetadata(), 'Unexpected bundle metadata.'); + + this.progress.bytesLoaded += element.byteLength; + + let documentsLoaded = this.progress.documentsLoaded; + + if (element.payload.namedQuery) { + this.queries.push(element.payload.namedQuery); + } else if (element.payload.documentMetadata) { + this.documents.push({ metadata: element.payload.documentMetadata }); + if (!element.payload.documentMetadata.exists) { + ++documentsLoaded; + } + } else if (element.payload.document) { + debugAssert( + this.documents.length > 0 && + this.documents[this.documents.length - 1].metadata.name === + element.payload.document.name, + 'The document being added does not match the stored metadata.' + ); + this.documents[this.documents.length - 1].document = + element.payload.document; + ++documentsLoaded; + } + + if (documentsLoaded !== this.progress.documentsLoaded) { + this.progress.documentsLoaded = documentsLoaded; + return { ...this.progress }; + } + + return null; + } + + private getQueryDocumentMapping( + documents: BundledDocuments + ): Map { + const queryDocumentMap = new Map(); + const bundleConverter = new BundleConverter(this.serializer); + for (const bundleDoc of documents) { + if (bundleDoc.metadata.queries) { + const documentKey = bundleConverter.toDocumentKey( + bundleDoc.metadata.name! + ); + for (const queryName of bundleDoc.metadata.queries) { + const documentKeys = ( + queryDocumentMap.get(queryName) || documentKeySet() + ).add(documentKey); + queryDocumentMap.set(queryName, documentKeys); + } + } + } + + return queryDocumentMap; + } + + /** + * Update the progress to 'Success' and return the updated progress. + */ + async complete(): Promise { + debugAssert( + this.documents[this.documents.length - 1]?.metadata.exists !== true || + !!this.documents[this.documents.length - 1].document, + 'Bundled documents ends with a document metadata and missing document.' + ); + debugAssert(!!this.bundleMetadata.id, 'Bundle ID must be set.'); + + const changedDocuments = await applyBundleDocuments( + this.localStore, + this.documents, + this.bundleMetadata.id! + ); + + const queryDocumentMap = this.getQueryDocumentMapping(this.documents); + + for (const q of this.queries) { + await saveNamedQuery(this.localStore, q, queryDocumentMap.get(q.name!)); + } + + this.progress.taskState = 'Success'; + return new BundleLoadResult({ ...this.progress }, changedDocuments); + } +} diff --git a/packages/firestore/src/core/component_provider.ts b/packages/firestore/src/core/component_provider.ts index 500e20dfcbb..bcd0f50e0b3 100644 --- a/packages/firestore/src/core/component_provider.ts +++ b/packages/firestore/src/core/component_provider.ts @@ -36,7 +36,8 @@ import { syncEngineHandleCredentialChange, newSyncEngine, SyncEngine, - ensureWriteCallbacks + ensureWriteCallbacks, + synchronizeWithChangedDocuments } from './sync_engine'; import { fillWritePipeline, @@ -67,6 +68,7 @@ import { newConnection, newConnectivityMonitor } from '../platform/connection'; import { newSerializer } from '../platform/serializer'; import { getDocument, getWindow } from '../platform/dom'; import { CredentialsProvider } from '../api/credentials'; +import { JsonProtoSerializer } from '../remote/serializer'; export interface ComponentConfiguration { asyncQueue: AsyncQueue; @@ -105,7 +107,10 @@ export class MemoryOfflineComponentProvider gcScheduler!: GarbageCollectionScheduler | null; synchronizeTabs = false; + serializer!: JsonProtoSerializer; + async initialize(cfg: ComponentConfiguration): Promise { + this.serializer = newSerializer(cfg.databaseInfo.databaseId); this.sharedClientState = this.createSharedClientState(cfg); this.persistence = this.createPersistence(cfg); await this.persistence.start(); @@ -123,12 +128,13 @@ export class MemoryOfflineComponentProvider return newLocalStore( this.persistence, new IndexFreeQueryEngine(), - cfg.initialUser + cfg.initialUser, + this.serializer ); } createPersistence(cfg: ComponentConfiguration): Persistence { - return new MemoryPersistence(MemoryEagerDelegate.factory); + return new MemoryPersistence(MemoryEagerDelegate.factory, this.serializer); } createSharedClientState(cfg: ComponentConfiguration): SharedClientState { @@ -173,6 +179,15 @@ export class IndexedDbOfflineComponentProvider extends MemoryOfflineComponentPro await fillWritePipeline(this.onlineComponentProvider.remoteStore); } + createLocalStore(cfg: ComponentConfiguration): LocalStore { + return newLocalStore( + this.persistence, + new IndexFreeQueryEngine(), + cfg.initialUser, + this.serializer + ); + } + createGarbageCollectionScheduler( cfg: ComponentConfiguration ): GarbageCollectionScheduler | null { @@ -190,7 +205,6 @@ export class IndexedDbOfflineComponentProvider extends MemoryOfflineComponentPro this.cacheSizeBytes !== undefined ? LruParams.withCacheSize(this.cacheSizeBytes) : LruParams.DEFAULT; - const serializer = newSerializer(cfg.databaseInfo.databaseId); return new IndexedDbPersistence( this.synchronizeTabs, @@ -200,7 +214,7 @@ export class IndexedDbOfflineComponentProvider extends MemoryOfflineComponentPro cfg.asyncQueue, getWindow(), getDocument(), - serializer, + this.serializer, this.sharedClientState, !!this.forceOwnership ); @@ -242,7 +256,11 @@ export class MultiTabOfflineComponentProvider extends IndexedDbOfflineComponentP null, syncEngine ), - getActiveClients: getActiveClients.bind(null, syncEngine) + getActiveClients: getActiveClients.bind(null, syncEngine), + synchronizeWithChangedDocuments: synchronizeWithChangedDocuments.bind( + null, + syncEngine + ) }; await this.sharedClientState.start(); } diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index eda44a6d622..c504017c429 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -23,6 +23,7 @@ import { } from '../api/credentials'; import { User } from '../auth/user'; import { + getNamedQuery, executeQuery, handleUserChange, LocalStore, @@ -54,6 +55,7 @@ import { registerPendingWritesCallback, SyncEngine, syncEngineListen, + syncEngineLoadBundle, syncEngineUnlisten, syncEngineWrite } from './sync_engine'; @@ -75,6 +77,12 @@ import { logDebug } from '../util/log'; import { AutoId } from '../util/misc'; import { Persistence } from '../local/persistence'; import { Datastore } from '../remote/datastore'; +import { BundleReader } from '../util/bundle_reader'; +import { LoadBundleTask } from '../api/bundle'; +import { newSerializer, newTextEncoder } from '../platform/serializer'; +import { toByteStreamReader } from '../platform/byte_stream_reader'; +import { NamedQuery } from './bundle'; +import { JsonProtoSerializer } from '../remote/serializer'; const LOG_TAG = 'FirestoreClient'; export const MAX_CONCURRENT_LIMBO_RESOLUTIONS = 100; @@ -648,3 +656,39 @@ export function executeQueryViaSnapshotListener( }); return eventManagerListen(eventManager, listener); } + +export async function firestoreClientLoadBundle( + client: FirestoreClient, + data: ReadableStream | ArrayBuffer | string, + resultTask: LoadBundleTask +): Promise { + const reader = createBundleReader( + data, + newSerializer((await client.getConfiguration()).databaseInfo.databaseId) + ); + client.asyncQueue.enqueueAndForget(async () => { + syncEngineLoadBundle(await getSyncEngine(client), reader, resultTask); + }); +} + +export function firestoreClientGetNamedQuery( + client: FirestoreClient, + queryName: string +): Promise { + return client.asyncQueue.enqueue(async () => + getNamedQuery(await getLocalStore(client), queryName) + ); +} + +function createBundleReader( + data: ReadableStream | ArrayBuffer | string, + serializer: JsonProtoSerializer +): BundleReader { + let content: ReadableStream | ArrayBuffer; + if (typeof data === 'string') { + content = newTextEncoder().encode(data); + } else { + content = data; + } + return new BundleReader(toByteStreamReader(content), serializer); +} diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 897ecbe0dea..b6bba5c1ac6 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -17,17 +17,19 @@ import { User } from '../auth/user'; import { + hasNewerBundle, + applyRemoteEventToLocalCache, getNewDocumentChanges, getCachedTarget, ignoreIfPrimaryLeaseLoss, LocalStore, + saveBundle, getActiveClientsFromPersistence, lookupMutationDocuments, removeCachedMutationBatchMetadata, allocateTarget, executeQuery, releaseTarget, - applyRemoteEventToLocalCache, rejectBatch, handleUserChange, localWrite, @@ -58,7 +60,7 @@ import { } from '../remote/remote_store'; import { debugAssert, debugCast, fail, hardAssert } from '../util/assert'; import { Code, FirestoreError } from '../util/error'; -import { logDebug } from '../util/log'; +import { logDebug, logWarn } from '../util/log'; import { primitiveComparator } from '../util/misc'; import { ObjectMap } from '../util/obj_map'; import { Deferred } from '../util/promise'; @@ -96,6 +98,13 @@ import { } from './view'; import { ViewSnapshot } from './view_snapshot'; import { wrapInUserErrorIfRecoverable } from '../util/async_queue'; +import { BundleReader } from '../util/bundle_reader'; +import { + BundleLoader, + bundleInitialProgress, + bundleSuccessProgress +} from './bundle'; +import { LoadBundleTask } from '../api/bundle'; import { EventManager, eventManagerOnOnlineStateChange, @@ -1150,6 +1159,21 @@ async function synchronizeViewAndComputeSnapshot( return viewSnapshot; } +/** + * Retrieves newly changed documents from remote document cache and raises + * snapshots if needed. + */ +// PORTING NOTE: Multi-Tab only. +export async function synchronizeWithChangedDocuments( + syncEngine: SyncEngine +): Promise { + const syncEngineImpl = debugCast(syncEngine, SyncEngineImpl); + + return getNewDocumentChanges(syncEngineImpl.localStore).then(changes => + emitNewSnapsAndNotifyLocalStore(syncEngineImpl, changes) + ); +} + /** Applies a mutation state to an existing batch. */ // PORTING NOTE: Multi-Tab only. export async function applyBatchState( @@ -1508,3 +1532,77 @@ export function ensureWriteCallbacks(syncEngine: SyncEngine): SyncEngineImpl { ); return syncEngineImpl; } + +/** + * Loads a Firestore bundle into the SDK. The returned promise resolves when + * the bundle finished loading. + * + * @param bundleReader Bundle to load into the SDK. + * @param task LoadBundleTask used to update the loading progress to public API. + */ +export function syncEngineLoadBundle( + syncEngine: SyncEngine, + bundleReader: BundleReader, + task: LoadBundleTask +): void { + const syncEngineImpl = debugCast(syncEngine, SyncEngineImpl); + + // eslint-disable-next-line @typescript-eslint/no-floating-promises + loadBundleImpl(syncEngineImpl, bundleReader, task).then(() => { + syncEngineImpl.sharedClientState.notifyBundleLoaded(); + }); +} + +async function loadBundleImpl( + syncEngine: SyncEngineImpl, + reader: BundleReader, + task: LoadBundleTask +): Promise { + try { + const metadata = await reader.getMetadata(); + const skip = await hasNewerBundle(syncEngine.localStore, metadata); + if (skip) { + await reader.close(); + task._completeWith(bundleSuccessProgress(metadata)); + return; + } + + task._updateProgress(bundleInitialProgress(metadata)); + + const loader = new BundleLoader( + metadata, + syncEngine.localStore, + reader.serializer + ); + let element = await reader.nextElement(); + while (element) { + debugAssert( + !element.payload.metadata, + 'Unexpected BundleMetadata element.' + ); + const progress = await loader.addSizedElement(element); + if (progress) { + task._updateProgress(progress); + } + + element = await reader.nextElement(); + } + + const result = await loader.complete(); + // TODO(b/160876443): This currently raises snapshots with + // `fromCache=false` if users already listen to some queries and bundles + // has newer version. + await emitNewSnapsAndNotifyLocalStore( + syncEngine, + result.changedDocs, + /* remoteEvent */ undefined + ); + + // Save metadata, so loading the same bundle will skip. + await saveBundle(syncEngine.localStore, metadata); + task._completeWith(result.progress); + } catch (e) { + logWarn(LOG_TAG, `Loading bundle failed with ${e}`); + task._failWith(e); + } +} diff --git a/packages/firestore/src/local/bundle_cache.ts b/packages/firestore/src/local/bundle_cache.ts new file mode 100644 index 00000000000..0bdd408e7b4 --- /dev/null +++ b/packages/firestore/src/local/bundle_cache.ts @@ -0,0 +1,61 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PersistenceTransaction } from './persistence'; +import { PersistencePromise } from './persistence_promise'; +import * as bundleProto from '../protos/firestore_bundle_proto'; +import { Bundle, NamedQuery } from '../core/bundle'; + +/** + * Provides interfaces to save and read Firestore bundles. + */ +export interface BundleCache { + /** + * Gets a saved `Bundle` for a given `bundleId`, returns undefined if + * no bundles are found under the given id. + */ + getBundleMetadata( + transaction: PersistenceTransaction, + bundleId: string + ): PersistencePromise; + + /** + * Saves a `BundleMetadata` from a bundle into local storage, using its id as + * the persistent key. + */ + saveBundleMetadata( + transaction: PersistenceTransaction, + metadata: bundleProto.BundleMetadata + ): PersistencePromise; + + /** + * Gets a saved `NamedQuery` for the given query name. Returns undefined if + * no queries are found under the given name. + */ + getNamedQuery( + transaction: PersistenceTransaction, + queryName: string + ): PersistencePromise; + + /** + * Saves a `NamedQuery` from a bundle, using its name as the persistent key. + */ + saveNamedQuery( + transaction: PersistenceTransaction, + query: bundleProto.NamedQuery + ): PersistencePromise; +} diff --git a/packages/firestore/src/local/indexeddb_bundle_cache.ts b/packages/firestore/src/local/indexeddb_bundle_cache.ts new file mode 100644 index 00000000000..d9384b160d9 --- /dev/null +++ b/packages/firestore/src/local/indexeddb_bundle_cache.ts @@ -0,0 +1,107 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PersistenceTransaction } from './persistence'; +import { PersistencePromise } from './persistence_promise'; +import * as bundleProto from '../protos/firestore_bundle_proto'; +import { BundleCache } from './bundle_cache'; +import { + DbBundle, + DbBundlesKey, + DbNamedQuery, + DbNamedQueriesKey +} from './indexeddb_schema'; +import { SimpleDbStore } from './simple_db'; +import { IndexedDbPersistence } from './indexeddb_persistence'; +import { + fromDbBundle, + fromDbNamedQuery, + LocalSerializer, + toDbBundle, + toDbNamedQuery +} from './local_serializer'; +import { Bundle, NamedQuery } from '../core/bundle'; + +export class IndexedDbBundleCache implements BundleCache { + constructor(private serializer: LocalSerializer) {} + + getBundleMetadata( + transaction: PersistenceTransaction, + bundleId: string + ): PersistencePromise { + return bundlesStore(transaction) + .get(bundleId) + .next(bundle => { + if (bundle) { + return fromDbBundle(bundle); + } + return undefined; + }); + } + + saveBundleMetadata( + transaction: PersistenceTransaction, + bundleMetadata: bundleProto.BundleMetadata + ): PersistencePromise { + return bundlesStore(transaction).put(toDbBundle(bundleMetadata)); + } + + getNamedQuery( + transaction: PersistenceTransaction, + queryName: string + ): PersistencePromise { + return namedQueriesStore(transaction) + .get(queryName) + .next(query => { + if (query) { + return fromDbNamedQuery(query); + } + return undefined; + }); + } + + saveNamedQuery( + transaction: PersistenceTransaction, + query: bundleProto.NamedQuery + ): PersistencePromise { + return namedQueriesStore(transaction).put(toDbNamedQuery(query)); + } +} + +/** + * Helper to get a typed SimpleDbStore for the bundles object store. + */ +function bundlesStore( + txn: PersistenceTransaction +): SimpleDbStore { + return IndexedDbPersistence.getStore( + txn, + DbBundle.store + ); +} + +/** + * Helper to get a typed SimpleDbStore for the namedQueries object store. + */ +function namedQueriesStore( + txn: PersistenceTransaction +): SimpleDbStore { + return IndexedDbPersistence.getStore( + txn, + DbNamedQuery.store + ); +} diff --git a/packages/firestore/src/local/indexeddb_persistence.ts b/packages/firestore/src/local/indexeddb_persistence.ts index 8ea569dd1e3..9982df38e94 100644 --- a/packages/firestore/src/local/indexeddb_persistence.ts +++ b/packages/firestore/src/local/indexeddb_persistence.ts @@ -30,6 +30,7 @@ import { EncodedResourcePath, encodeResourcePath } from './encoded_resource_path'; +import { IndexedDbBundleCache } from './indexeddb_bundle_cache'; import { IndexedDbIndexManager } from './indexeddb_index_manager'; import { IndexedDbMutationQueue, @@ -222,6 +223,7 @@ export class IndexedDbPersistence implements Persistence { private readonly targetCache: IndexedDbTargetCache; private readonly indexManager: IndexedDbIndexManager; private readonly remoteDocumentCache: IndexedDbRemoteDocumentCache; + private readonly bundleCache: IndexedDbBundleCache; private readonly webStorage: Storage | null; readonly referenceDelegate: IndexedDbLruDelegate; @@ -271,6 +273,7 @@ export class IndexedDbPersistence implements Persistence { this.serializer, this.indexManager ); + this.bundleCache = new IndexedDbBundleCache(this.serializer); if (this.window && this.window.localStorage) { this.webStorage = this.window.localStorage; } else { @@ -780,6 +783,14 @@ export class IndexedDbPersistence implements Persistence { return this.indexManager; } + getBundleCache(): IndexedDbBundleCache { + debugAssert( + this.started, + 'Cannot initialize BundleCache before persistence is started.' + ); + return this.bundleCache; + } + runTransaction( action: string, mode: PersistenceTransactionMode, diff --git a/packages/firestore/src/local/indexeddb_remote_document_cache.ts b/packages/firestore/src/local/indexeddb_remote_document_cache.ts index 56e0cffd7b1..e4c50cb4901 100644 --- a/packages/firestore/src/local/indexeddb_remote_document_cache.ts +++ b/packages/firestore/src/local/indexeddb_remote_document_cache.ts @@ -485,21 +485,21 @@ class IndexedDbRemoteDocumentChangeBuffer extends RemoteDocumentChangeBuffer { primitiveComparator(l.canonicalString(), r.canonicalString()) ); - this.changes.forEach((key, maybeDocument) => { + this.changes.forEach((key, documentChange) => { const previousSize = this.documentSizes.get(key); debugAssert( previousSize !== undefined, `Cannot modify a document that wasn't read (for ${key})` ); - if (maybeDocument) { + if (documentChange.maybeDocument) { debugAssert( - !this.readTime.isEqual(SnapshotVersion.min()), + !this.getReadTime(key).isEqual(SnapshotVersion.min()), 'Cannot add a document with a read time of zero' ); const doc = toDbRemoteDocument( this.documentCache.serializer, - maybeDocument, - this.readTime + documentChange.maybeDocument, + this.getReadTime(key) ); collectionParents = collectionParents.add(key.path.popLast()); @@ -516,7 +516,7 @@ class IndexedDbRemoteDocumentChangeBuffer extends RemoteDocumentChangeBuffer { const deletedDoc = toDbRemoteDocument( this.documentCache.serializer, new NoDocument(key, SnapshotVersion.min()), - this.readTime + this.getReadTime(key) ); promises.push( this.documentCache.addEntry(transaction, key, deletedDoc) diff --git a/packages/firestore/src/local/indexeddb_schema.ts b/packages/firestore/src/local/indexeddb_schema.ts index 9e2403542b0..80eb9cd4c49 100644 --- a/packages/firestore/src/local/indexeddb_schema.ts +++ b/packages/firestore/src/local/indexeddb_schema.ts @@ -17,6 +17,7 @@ import { BatchId, ListenSequenceNumber, TargetId } from '../core/types'; import { ResourcePath } from '../model/path'; +import { BundledQuery } from '../protos/firestore_bundle_proto'; import { Write as ProtoWrite, Document as ProtoDocument, @@ -61,8 +62,9 @@ import { SimpleDbSchemaConverter, SimpleDbTransaction } from './simple_db'; * 9. Change RemoteDocumentChanges store to be keyed by readTime rather than * an auto-incrementing ID. This is required for Index-Free queries. * 10. Rewrite the canonical IDs to the explicit Protobuf-based format. + * 11. Add bundles and named_queries for bundle support. */ -export const SCHEMA_VERSION = 10; +export const SCHEMA_VERSION = 11; /** Performs database creation and schema upgrades. */ export class SchemaConverter implements SimpleDbSchemaConverter { @@ -163,6 +165,13 @@ export class SchemaConverter implements SimpleDbSchemaConverter { if (fromVersion < 10 && toVersion >= 10) { p = p.next(() => this.rewriteCanonicalIds(simpleDbTransaction)); } + + if (fromVersion < 11 && toVersion >= 11) { + p = p.next(() => { + createBundlesStore(db); + createNamedQueriesStore(db); + }); + } return p; } @@ -1090,6 +1099,60 @@ function createClientMetadataStore(db: IDBDatabase): void { }); } +export type DbBundlesKey = string; + +/** + * A object representing a bundle loaded by the SDK. + */ +export class DbBundle { + /** Name of the IndexedDb object store. */ + static store = 'bundles'; + + static keyPath = 'bundleId'; + + constructor( + /** The ID of the loaded bundle. */ + public bundleId: string, + /** The create time of the loaded bundle. */ + public createTime: DbTimestamp, + /** The schema version of the loaded bundle. */ + public version: number + ) {} +} + +function createBundlesStore(db: IDBDatabase): void { + db.createObjectStore(DbBundle.store, { + keyPath: DbBundle.keyPath + }); +} + +export type DbNamedQueriesKey = string; + +/** + * A object representing a named query loaded by the SDK via a bundle. + */ +export class DbNamedQuery { + /** Name of the IndexedDb object store. */ + static store = 'namedQueries'; + + static keyPath = 'name'; + + constructor( + /** The name of the query. */ + public name: string, + /** The read time of the results saved in the bundle from the named query. */ + public readTime: DbTimestamp, + /** The query saved in the bundle. */ + public bundledQuery: BundledQuery + ) {} +} + +function createNamedQueriesStore(db: IDBDatabase): void { + db.createObjectStore(DbNamedQuery.store, { + keyPath: DbNamedQuery.keyPath + }); +} + // Visible for testing export const V1_STORES = [ DbMutationQueue.store, @@ -1123,9 +1186,11 @@ export const V8_STORES = [...V6_STORES, DbCollectionParent.store]; // V10 does not change the set of stores. +export const V11_STORES = [...V8_STORES, DbBundle.store, DbNamedQuery.store]; + /** * The list of all default IndexedDB stores used throughout the SDK. This is * used when creating transactions so that access across all stores is done * atomically. */ -export const ALL_STORES = V8_STORES; +export const ALL_STORES = V11_STORES; diff --git a/packages/firestore/src/local/local_serializer.ts b/packages/firestore/src/local/local_serializer.ts index 69662f585e0..1cc80bc2ccd 100644 --- a/packages/firestore/src/local/local_serializer.ts +++ b/packages/firestore/src/local/local_serializer.ts @@ -27,10 +27,12 @@ import { DocumentKey } from '../model/document_key'; import { MutationBatch } from '../model/mutation_batch'; import { DocumentsTarget as PublicDocumentsTarget } from '../protos/firestore_proto_api'; import { + convertQueryTargetToQuery, fromDocument, fromDocumentsTarget, fromMutation, fromQueryTarget, + fromVersion, JsonProtoSerializer, toDocument, toDocumentsTarget, @@ -41,7 +43,9 @@ import { debugAssert, fail } from '../util/assert'; import { ByteString } from '../util/byte_string'; import { canonifyTarget, isDocumentTarget, Target } from '../core/target'; import { + DbBundle, DbMutationBatch, + DbNamedQuery, DbNoDocument, DbQuery, DbRemoteDocument, @@ -51,6 +55,9 @@ import { DbUnknownDocument } from './indexeddb_schema'; import { TargetData, TargetPurpose } from './target_data'; +import { Bundle, NamedQuery } from '../core/bundle'; +import { LimitType, Query, queryWithLimit } from '../core/query'; +import * as bundleProto from '../protos/firestore_bundle_proto'; /** Serializer for values stored in the LocalStore. */ export class LocalSerializer { @@ -271,3 +278,84 @@ export function toDbTarget( function isDocumentQuery(dbQuery: DbQuery): dbQuery is PublicDocumentsTarget { return (dbQuery as PublicDocumentsTarget).documents !== undefined; } + +/** Encodes a DbBundle to a Bundle. */ +export function fromDbBundle(dbBundle: DbBundle): Bundle { + return { + id: dbBundle.bundleId, + createTime: fromDbTimestamp(dbBundle.createTime), + version: dbBundle.version + }; +} + +/** Encodes a BundleMetadata to a DbBundle. */ +export function toDbBundle(metadata: bundleProto.BundleMetadata): DbBundle { + return { + bundleId: metadata.id!, + createTime: toDbTimestamp(fromVersion(metadata.createTime!)), + version: metadata.version! + }; +} + +/** Encodes a DbNamedQuery to a NamedQuery. */ +export function fromDbNamedQuery(dbNamedQuery: DbNamedQuery): NamedQuery { + return { + name: dbNamedQuery.name, + query: fromBundledQuery(dbNamedQuery.bundledQuery), + readTime: fromDbTimestamp(dbNamedQuery.readTime) + }; +} + +/** Encodes a NamedQuery from a bundle proto to a DbNamedQuery. */ +export function toDbNamedQuery(query: bundleProto.NamedQuery): DbNamedQuery { + return { + name: query.name!, + readTime: toDbTimestamp(fromVersion(query.readTime!)), + bundledQuery: query.bundledQuery! + }; +} + +/** + * Encodes a `BundledQuery` from bundle proto to a Query object. + * + * This reconstructs the original query used to build the bundle being loaded, + * including features exists only in SDKs (for example: limit-to-last). + */ +export function fromBundledQuery( + bundledQuery: bundleProto.BundledQuery +): Query { + const query = convertQueryTargetToQuery({ + parent: bundledQuery.parent!, + structuredQuery: bundledQuery.structuredQuery! + }); + if (bundledQuery.limitType === 'LAST') { + debugAssert( + !!query.limit, + 'Bundled query has limitType LAST, but limit is null' + ); + return queryWithLimit(query, query.limit, LimitType.Last); + } + return query; +} + +/** Encodes a NamedQuery proto object to a NamedQuery model object. */ +export function fromProtoNamedQuery( + namedQuery: bundleProto.NamedQuery +): NamedQuery { + return { + name: namedQuery.name!, + query: fromBundledQuery(namedQuery.bundledQuery!), + readTime: fromVersion(namedQuery.readTime!) + }; +} + +/** Encodes a BundleMetadata proto object to a Bundle model object. */ +export function fromBundleMetadata( + metadata: bundleProto.BundleMetadata +): Bundle { + return { + id: metadata.id!, + version: metadata.version!, + createTime: fromVersion(metadata.createTime!) + }; +} diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 630dbf9f962..576cece89fc 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -17,7 +17,7 @@ import { Timestamp } from '../api/timestamp'; import { User } from '../auth/user'; -import { Query, queryToTarget } from '../core/query'; +import { newQueryForPath, Query, queryToTarget } from '../core/query'; import { SnapshotVersion } from '../core/snapshot_version'; import { canonifyTarget, Target, targetEquals } from '../core/target'; import { BatchId, TargetId } from '../core/types'; @@ -25,16 +25,18 @@ import { DocumentKeySet, documentKeySet, DocumentMap, + documentVersionMap, + DocumentVersionMap, maybeDocumentMap, MaybeDocumentMap } from '../model/collections'; import { MaybeDocument, NoDocument } from '../model/document'; import { DocumentKey } from '../model/document_key'; import { + extractMutationBaseValue, Mutation, PatchMutation, - Precondition, - extractMutationBaseValue + Precondition } from '../model/mutation'; import { BATCHID_UNKNOWN, @@ -74,6 +76,13 @@ import { import { IndexedDbTargetCache } from './indexeddb_target_cache'; import { extractFieldMask } from '../model/object_value'; import { isIndexedDbTransactionError } from './simple_db'; +import * as bundleProto from '../protos/firestore_bundle_proto'; +import { BundleConverter, BundledDocuments, NamedQuery } from '../core/bundle'; +import { BundleCache } from './bundle_cache'; +import { fromVersion, JsonProtoSerializer } from '../remote/serializer'; +import { fromBundledQuery } from './local_serializer'; +import { ByteString } from '../util/byte_string'; +import { ResourcePath } from '../model/path'; const LOG_TAG = 'LocalStore'; @@ -187,6 +196,9 @@ class LocalStoreImpl implements LocalStore { */ localDocuments: LocalDocumentsView; + /** The set of all cached bundle metadata and named queries. */ + bundleCache: BundleCache; + /** Maps a target to its `TargetData`. */ targetCache: TargetCache; @@ -216,7 +228,8 @@ class LocalStoreImpl implements LocalStore { /** Manages our in-memory or durable persistence. */ readonly persistence: Persistence, readonly queryEngine: QueryEngine, - initialUser: User + initialUser: User, + readonly serializer: JsonProtoSerializer ) { debugAssert( persistence.started, @@ -230,6 +243,7 @@ class LocalStoreImpl implements LocalStore { this.mutationQueue, this.persistence.getIndexManager() ); + this.bundleCache = persistence.getBundleCache(); this.queryEngine.setLocalDocumentsView(this.localDocuments); } @@ -246,9 +260,10 @@ export function newLocalStore( /** Manages our in-memory or durable persistence. */ persistence: Persistence, queryEngine: QueryEngine, - initialUser: User + initialUser: User, + serializer: JsonProtoSerializer ): LocalStore { - return new LocalStoreImpl(persistence, queryEngine, initialUser); + return new LocalStoreImpl(persistence, queryEngine, initialUser, serializer); } /** @@ -569,64 +584,28 @@ export function applyRemoteEventToLocalCache( }); let changedDocs = maybeDocumentMap(); - let updatedKeys = documentKeySet(); remoteEvent.documentUpdates.forEach((key, doc) => { - updatedKeys = updatedKeys.add(key); + if (remoteEvent.resolvedLimboDocuments.has(key)) { + promises.push( + localStoreImpl.persistence.referenceDelegate.updateLimboDocument( + txn, + key + ) + ); + } }); // Each loop iteration only affects its "own" doc, so it's safe to get all the remote // documents in advance in a single call. promises.push( - documentBuffer.getEntries(txn, updatedKeys).next(existingDocs => { - remoteEvent.documentUpdates.forEach((key, doc) => { - const existingDoc = existingDocs.get(key); - - // Note: The order of the steps below is important, since we want - // to ensure that rejected limbo resolutions (which fabricate - // NoDocuments with SnapshotVersion.min()) never add documents to - // cache. - if ( - doc instanceof NoDocument && - doc.version.isEqual(SnapshotVersion.min()) - ) { - // NoDocuments with SnapshotVersion.min() are used in manufactured - // events. We remove these documents from cache since we lost - // access. - documentBuffer.removeEntry(key, remoteVersion); - changedDocs = changedDocs.insert(key, doc); - } else if ( - existingDoc == null || - doc.version.compareTo(existingDoc.version) > 0 || - (doc.version.compareTo(existingDoc.version) === 0 && - existingDoc.hasPendingWrites) - ) { - debugAssert( - !SnapshotVersion.min().isEqual(remoteVersion), - 'Cannot add a document when the remote version is zero' - ); - documentBuffer.addEntry(doc, remoteVersion); - changedDocs = changedDocs.insert(key, doc); - } else { - logDebug( - LOG_TAG, - 'Ignoring outdated watch update for ', - key, - '. Current version:', - existingDoc.version, - ' Watch version:', - doc.version - ); - } - - if (remoteEvent.resolvedLimboDocuments.has(key)) { - promises.push( - localStoreImpl.persistence.referenceDelegate.updateLimboDocument( - txn, - key - ) - ); - } - }); + populateDocumentChangeBuffer( + txn, + documentBuffer, + remoteEvent.documentUpdates, + remoteVersion, + undefined + ).next(result => { + changedDocs = result; }) ); @@ -669,6 +648,80 @@ export function applyRemoteEventToLocalCache( }); } +/** + * Populates document change buffer with documents from backend or a bundle. + * Returns the document changes resulting from applying those documents. + * + * @param txn Transaction to use to read existing documents from storage. + * @param documentBuffer Document buffer to collect the resulted changes to be + * applied to storage. + * @param documents Documents to be applied. + * @param globalVersion A `SnapshotVersion` representing the read time if all + * documents have the same read time. + * @param documentVersions A DocumentKey-to-SnapshotVersion map if documents + * have their own read time. + * + * Note: this function will use `documentVersions` if it is defined; + * when it is not defined, resorts to `globalVersion`. + */ +function populateDocumentChangeBuffer( + txn: PersistenceTransaction, + documentBuffer: RemoteDocumentChangeBuffer, + documents: MaybeDocumentMap, + globalVersion: SnapshotVersion, + // TODO(wuandy): We could add `readTime` to MaybeDocument instead to remove + // this parameter. + documentVersions: DocumentVersionMap | undefined +): PersistencePromise { + let updatedKeys = documentKeySet(); + documents.forEach(k => (updatedKeys = updatedKeys.add(k))); + return documentBuffer.getEntries(txn, updatedKeys).next(existingDocs => { + let changedDocs = maybeDocumentMap(); + documents.forEach((key, doc) => { + const existingDoc = existingDocs.get(key); + const docReadTime = documentVersions?.get(key) || globalVersion; + + // Note: The order of the steps below is important, since we want + // to ensure that rejected limbo resolutions (which fabricate + // NoDocuments with SnapshotVersion.min()) never add documents to + // cache. + if ( + doc instanceof NoDocument && + doc.version.isEqual(SnapshotVersion.min()) + ) { + // NoDocuments with SnapshotVersion.min() are used in manufactured + // events. We remove these documents from cache since we lost + // access. + documentBuffer.removeEntry(key, docReadTime); + changedDocs = changedDocs.insert(key, doc); + } else if ( + existingDoc == null || + doc.version.compareTo(existingDoc.version) > 0 || + (doc.version.compareTo(existingDoc.version) === 0 && + existingDoc.hasPendingWrites) + ) { + debugAssert( + !SnapshotVersion.min().isEqual(docReadTime), + 'Cannot add a document when the remote version is zero' + ); + documentBuffer.addEntry(doc, docReadTime); + changedDocs = changedDocs.insert(key, doc); + } else { + logDebug( + LOG_TAG, + 'Ignoring outdated watch update for ', + key, + '. Current version:', + existingDoc.version, + ' Watch version:', + doc.version + ); + } + }); + return changedDocs; + }); +} + /** * Returns true if the newTargetData should be persisted during an update of * an active target. TargetData should always be persisted when a target is @@ -1218,3 +1271,219 @@ export async function ignoreIfPrimaryLeaseLoss( throw err; } } + +/** + * Creates a new target using the given bundle name, which will be used to + * hold the keys of all documents from the bundle in query-document mappings. + * This ensures that the loaded documents do not get garbage collected + * right away. + */ +function umbrellaTarget(bundleName: string): Target { + // It is OK that the path used for the query is not valid, because this will + // not be read and queried. + return queryToTarget( + newQueryForPath(ResourcePath.fromString(`__bundle__/docs/${bundleName}`)) + ); +} + +/** + * Applies the documents from a bundle to the "ground-state" (remote) + * documents. + * + * LocalDocuments are re-calculated if there are remaining mutations in the + * queue. + */ +export async function applyBundleDocuments( + localStore: LocalStore, + documents: BundledDocuments, + bundleName: string +): Promise { + const localStoreImpl = debugCast(localStore, LocalStoreImpl); + const bundleConverter = new BundleConverter(localStoreImpl.serializer); + let documentKeys = documentKeySet(); + let documentMap = maybeDocumentMap(); + let versionMap = documentVersionMap(); + for (const bundleDoc of documents) { + const documentKey = bundleConverter.toDocumentKey(bundleDoc.metadata.name!); + if (bundleDoc.document) { + documentKeys = documentKeys.add(documentKey); + } + documentMap = documentMap.insert( + documentKey, + bundleConverter.toMaybeDocument(bundleDoc) + ); + versionMap = versionMap.insert( + documentKey, + bundleConverter.toSnapshotVersion(bundleDoc.metadata.readTime!) + ); + } + + const documentBuffer = localStoreImpl.remoteDocuments.newChangeBuffer({ + trackRemovals: true // Make sure document removals show up in `getNewDocumentChanges()` + }); + + // Allocates a target to hold all document keys from the bundle, such that + // they will not get garbage collected right away. + const umbrellaTargetData = await allocateTarget( + localStoreImpl, + umbrellaTarget(bundleName) + ); + return localStoreImpl.persistence.runTransaction( + 'Apply bundle documents', + 'readwrite', + txn => { + return populateDocumentChangeBuffer( + txn, + documentBuffer, + documentMap, + SnapshotVersion.min(), + versionMap + ) + .next(changedDocs => { + documentBuffer.apply(txn); + return changedDocs; + }) + .next(changedDocs => { + return localStoreImpl.targetCache + .removeMatchingKeysForTargetId(txn, umbrellaTargetData.targetId) + .next(() => + localStoreImpl.targetCache.addMatchingKeys( + txn, + documentKeys, + umbrellaTargetData.targetId + ) + ) + .next(() => + localStoreImpl.localDocuments.getLocalViewOfDocuments( + txn, + changedDocs + ) + ); + }); + } + ); +} + +/** + * Returns a promise of a boolean to indicate if the given bundle has already + * been loaded and the create time is newer than the current loading bundle. + */ +export function hasNewerBundle( + localStore: LocalStore, + bundleMetadata: bundleProto.BundleMetadata +): Promise { + const localStoreImpl = debugCast(localStore, LocalStoreImpl); + const bundleConverter = new BundleConverter(localStoreImpl.serializer); + const currentReadTime = bundleConverter.toSnapshotVersion( + bundleMetadata.createTime! + ); + return localStoreImpl.persistence + .runTransaction('hasNewerBundle', 'readonly', transaction => { + return localStoreImpl.bundleCache.getBundleMetadata( + transaction, + bundleMetadata.id! + ); + }) + .then(cached => { + return !!cached && cached.createTime!.compareTo(currentReadTime) >= 0; + }); +} + +/** + * Saves the given `BundleMetadata` to local persistence. + * @param bundleMetadata + */ +export function saveBundle( + localStore: LocalStore, + bundleMetadata: bundleProto.BundleMetadata +): Promise { + const localStoreImpl = debugCast(localStore, LocalStoreImpl); + return localStoreImpl.persistence.runTransaction( + 'Save bundle', + 'readwrite', + transaction => { + return localStoreImpl.bundleCache.saveBundleMetadata( + transaction, + bundleMetadata + ); + } + ); +} + +/** + * Returns a promise of a `NamedQuery` associated with given query name. Promise + * resolves to undefined if no persisted data can be found. + */ +export function getNamedQuery( + localStore: LocalStore, + queryName: string +): Promise { + const localStoreImpl = debugCast(localStore, LocalStoreImpl); + return localStoreImpl.persistence.runTransaction( + 'Get named query', + 'readonly', + transaction => + localStoreImpl.bundleCache.getNamedQuery(transaction, queryName) + ); +} + +/** + * Saves the given `NamedQuery` to local persistence. + */ +export async function saveNamedQuery( + localStore: LocalStore, + query: bundleProto.NamedQuery, + documents: DocumentKeySet = documentKeySet() +): Promise { + // Allocate a target for the named query such that it can be resumed + // from associated read time if users use it to listen. + // NOTE: this also means if no corresponding target exists, the new target + // will remain active and will not get collected, unless users happen to + // unlisten the query somehow. + const allocated = await allocateTarget( + localStore, + queryToTarget(fromBundledQuery(query.bundledQuery!)) + ); + + const localStoreImpl = debugCast(localStore, LocalStoreImpl); + return localStoreImpl.persistence.runTransaction( + 'Save named query', + 'readwrite', + transaction => { + const readTime = fromVersion(query.readTime!); + // Simply save the query itself if it is older than what the SDK already + // has. + if (allocated.snapshotVersion.compareTo(readTime) >= 0) { + return localStoreImpl.bundleCache.saveNamedQuery(transaction, query); + } + + // Update existing target data because the query from the bundle is newer. + const newTargetData = allocated.withResumeToken( + ByteString.EMPTY_BYTE_STRING, + readTime + ); + localStoreImpl.targetDataByTarget = localStoreImpl.targetDataByTarget.insert( + newTargetData.targetId, + newTargetData + ); + return localStoreImpl.targetCache + .updateTargetData(transaction, newTargetData) + .next(() => + localStoreImpl.targetCache.removeMatchingKeysForTargetId( + transaction, + allocated.targetId + ) + ) + .next(() => + localStoreImpl.targetCache.addMatchingKeys( + transaction, + documents, + allocated.targetId + ) + ) + .next(() => + localStoreImpl.bundleCache.saveNamedQuery(transaction, query) + ); + } + ); +} diff --git a/packages/firestore/src/local/memory_bundle_cache.ts b/packages/firestore/src/local/memory_bundle_cache.ts new file mode 100644 index 00000000000..788382ca387 --- /dev/null +++ b/packages/firestore/src/local/memory_bundle_cache.ts @@ -0,0 +1,64 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PersistenceTransaction } from './persistence'; +import { PersistencePromise } from './persistence_promise'; +import * as bundleProto from '../protos/firestore_bundle_proto'; +import { BundleCache } from './bundle_cache'; +import { Bundle, NamedQuery } from '../core/bundle'; +import { + fromBundleMetadata, + fromProtoNamedQuery, + LocalSerializer +} from './local_serializer'; + +export class MemoryBundleCache implements BundleCache { + private bundles = new Map(); + private namedQueries = new Map(); + + constructor(private serializer: LocalSerializer) {} + + getBundleMetadata( + transaction: PersistenceTransaction, + bundleId: string + ): PersistencePromise { + return PersistencePromise.resolve(this.bundles.get(bundleId)); + } + + saveBundleMetadata( + transaction: PersistenceTransaction, + bundleMetadata: bundleProto.BundleMetadata + ): PersistencePromise { + this.bundles.set(bundleMetadata.id!, fromBundleMetadata(bundleMetadata)); + return PersistencePromise.resolve(); + } + + getNamedQuery( + transaction: PersistenceTransaction, + queryName: string + ): PersistencePromise { + return PersistencePromise.resolve(this.namedQueries.get(queryName)); + } + + saveNamedQuery( + transaction: PersistenceTransaction, + query: bundleProto.NamedQuery + ): PersistencePromise { + this.namedQueries.set(query.name!, fromProtoNamedQuery(query)); + return PersistencePromise.resolve(); + } +} diff --git a/packages/firestore/src/local/memory_persistence.ts b/packages/firestore/src/local/memory_persistence.ts index 36219c08f4d..ce6e2c4b209 100644 --- a/packages/firestore/src/local/memory_persistence.ts +++ b/packages/firestore/src/local/memory_persistence.ts @@ -48,6 +48,9 @@ import { import { PersistencePromise } from './persistence_promise'; import { ReferenceSet } from './reference_set'; import { TargetData } from './target_data'; +import { MemoryBundleCache } from './memory_bundle_cache'; +import { JsonProtoSerializer } from '../remote/serializer'; +import { LocalSerializer } from './local_serializer'; const LOG_TAG = 'MemoryPersistence'; /** @@ -66,7 +69,9 @@ export class MemoryPersistence implements Persistence { private mutationQueues: { [user: string]: MemoryMutationQueue } = {}; private readonly remoteDocumentCache: MemoryRemoteDocumentCache; private readonly targetCache: MemoryTargetCache; + private readonly bundleCache: MemoryBundleCache; private readonly listenSequence = new ListenSequence(0); + private serializer: LocalSerializer; private _started = false; @@ -79,7 +84,8 @@ export class MemoryPersistence implements Persistence { * checked or asserted on every access. */ constructor( - referenceDelegateFactory: (p: MemoryPersistence) => MemoryReferenceDelegate + referenceDelegateFactory: (p: MemoryPersistence) => MemoryReferenceDelegate, + serializer: JsonProtoSerializer ) { this._started = true; this.referenceDelegate = referenceDelegateFactory(this); @@ -91,6 +97,8 @@ export class MemoryPersistence implements Persistence { this.indexManager, sizer ); + this.serializer = new LocalSerializer(serializer); + this.bundleCache = new MemoryBundleCache(this.serializer); } start(): Promise { @@ -139,6 +147,10 @@ export class MemoryPersistence implements Persistence { return this.remoteDocumentCache; } + getBundleCache(): MemoryBundleCache { + return this.bundleCache; + } + runTransaction( action: string, mode: PersistenceTransactionMode, diff --git a/packages/firestore/src/local/memory_remote_document_cache.ts b/packages/firestore/src/local/memory_remote_document_cache.ts index e60dfd8bea4..b9cb74ad101 100644 --- a/packages/firestore/src/local/memory_remote_document_cache.ts +++ b/packages/firestore/src/local/memory_remote_document_cache.ts @@ -229,9 +229,13 @@ class MemoryRemoteDocumentChangeBuffer extends RemoteDocumentChangeBuffer { ): PersistencePromise { const promises: Array> = []; this.changes.forEach((key, doc) => { - if (doc) { + if (doc && doc.maybeDocument) { promises.push( - this.documentCache.addEntry(transaction, doc, this.readTime) + this.documentCache.addEntry( + transaction, + doc.maybeDocument!, + this.getReadTime(key) + ) ); } else { this.documentCache.removeEntry(key); diff --git a/packages/firestore/src/local/persistence.ts b/packages/firestore/src/local/persistence.ts index 365bfee8c0c..6a4f541796e 100644 --- a/packages/firestore/src/local/persistence.ts +++ b/packages/firestore/src/local/persistence.ts @@ -25,6 +25,7 @@ import { PersistencePromise } from './persistence_promise'; import { TargetCache } from './target_cache'; import { RemoteDocumentCache } from './remote_document_cache'; import { TargetData } from './target_data'; +import { BundleCache } from './bundle_cache'; export const PRIMARY_LEASE_LOST_ERROR_MSG = 'The current tab is not in the required state to perform this operation. ' + @@ -224,6 +225,15 @@ export interface Persistence { */ getRemoteDocumentCache(): RemoteDocumentCache; + /** + * Returns a BundleCache representing the persisted cache of loaded bundles. + * + * Note: The implementation is free to return the same instance every time + * this is called. In particular, the memory-backed implementation does this + * to emulate the persisted implementation to the extent possible. + */ + getBundleCache(): BundleCache; + /** * Returns an IndexManager instance that manages our persisted query indexes. * diff --git a/packages/firestore/src/local/remote_document_change_buffer.ts b/packages/firestore/src/local/remote_document_change_buffer.ts index c54a0a0a3e7..cc414d402a4 100644 --- a/packages/firestore/src/local/remote_document_change_buffer.ts +++ b/packages/firestore/src/local/remote_document_change_buffer.ts @@ -25,6 +25,16 @@ import { PersistenceTransaction } from './persistence'; import { PersistencePromise } from './persistence_promise'; import { SnapshotVersion } from '../core/snapshot_version'; +/** + * Represents a document change to be applied to remote document cache. + */ +interface RemoteDocumentChange { + // The document in this change, null if it is a removal from the cache. + readonly maybeDocument: MaybeDocument | null; + // The timestamp when this change is read. + readonly readTime: SnapshotVersion | null; +} + /** * An in-memory buffer of entries to be written to a RemoteDocumentCache. * It can be used to batch up a set of changes to be written to the cache, but @@ -44,15 +54,12 @@ export abstract class RemoteDocumentChangeBuffer { // existing cache entry should be removed). protected changes: ObjectMap< DocumentKey, - MaybeDocument | null + RemoteDocumentChange > = new ObjectMap( key => key.toString(), (l, r) => l.isEqual(r) ); - // The read time to use for all added documents in this change buffer. - private _readTime: SnapshotVersion | undefined; - private changesApplied = false; protected abstract getFromCache( @@ -69,23 +76,16 @@ export abstract class RemoteDocumentChangeBuffer { transaction: PersistenceTransaction ): PersistencePromise; - protected set readTime(value: SnapshotVersion) { - // Right now (for simplicity) we just track a single readTime for all the - // added entries since we expect them to all be the same, but we could - // rework to store per-entry readTimes if necessary. - debugAssert( - this._readTime === undefined || this._readTime.isEqual(value), - 'All changes in a RemoteDocumentChangeBuffer must have the same read time' - ); - this._readTime = value; - } - - protected get readTime(): SnapshotVersion { - debugAssert( - this._readTime !== undefined, - 'Read time is not set. All removeEntry() calls must include a readTime if `trackRemovals` is used.' - ); - return this._readTime; + protected getReadTime(key: DocumentKey): SnapshotVersion { + const change = this.changes.get(key); + if (change) { + debugAssert( + !!change.readTime, + `Read time is not set for ${key}. All removeEntry() calls must include a readTime if 'trackRemovals' is used.` + ); + return change.readTime; + } + return SnapshotVersion.min(); } /** @@ -96,8 +96,7 @@ export abstract class RemoteDocumentChangeBuffer { */ addEntry(maybeDocument: MaybeDocument, readTime: SnapshotVersion): void { this.assertNotApplied(); - this.readTime = readTime; - this.changes.set(maybeDocument.key, maybeDocument); + this.changes.set(maybeDocument.key, { maybeDocument, readTime }); } /** @@ -106,12 +105,9 @@ export abstract class RemoteDocumentChangeBuffer { * You can only remove documents that have already been retrieved via * `getEntry()/getEntries()` (enforced via IndexedDbs `apply()`). */ - removeEntry(key: DocumentKey, readTime?: SnapshotVersion): void { + removeEntry(key: DocumentKey, readTime: SnapshotVersion | null = null): void { this.assertNotApplied(); - if (readTime) { - this.readTime = readTime; - } - this.changes.set(key, null); + this.changes.set(key, { maybeDocument: null, readTime }); } /** @@ -132,7 +128,9 @@ export abstract class RemoteDocumentChangeBuffer { this.assertNotApplied(); const bufferedEntry = this.changes.get(documentKey); if (bufferedEntry !== undefined) { - return PersistencePromise.resolve(bufferedEntry); + return PersistencePromise.resolve( + bufferedEntry.maybeDocument + ); } else { return this.getFromCache(transaction, documentKey); } diff --git a/packages/firestore/src/local/shared_client_state.ts b/packages/firestore/src/local/shared_client_state.ts index 03815ce7095..9d77553a77e 100644 --- a/packages/firestore/src/local/shared_client_state.ts +++ b/packages/firestore/src/local/shared_client_state.ts @@ -40,6 +40,7 @@ import { import { CLIENT_STATE_KEY_PREFIX, ClientStateSchema, + createBundleLoadedKey, createWebStorageClientStateKey, createWebStorageMutationBatchKey, createWebStorageOnlineStateKey, @@ -172,6 +173,12 @@ export interface SharedClientState { setOnlineState(onlineState: OnlineState): void; writeSequenceNumber(sequenceNumber: ListenSequenceNumber): void; + + /** + * Notifies other clients when remote documents have changed due to loading + * a bundle. + */ + notifyBundleLoaded(): void; } /** @@ -476,6 +483,7 @@ export class WebStorageSharedClientState implements SharedClientState { private readonly sequenceNumberKey: string; private readonly storageListener = this.handleWebStorageEvent.bind(this); private readonly onlineStateKey: string; + private readonly bundleLoadedKey: string; private readonly clientStateKeyRe: RegExp; private readonly mutationBatchKeyRe: RegExp; private readonly queryTargetKeyRe: RegExp; @@ -531,6 +539,8 @@ export class WebStorageSharedClientState implements SharedClientState { this.onlineStateKey = createWebStorageOnlineStateKey(this.persistenceKey); + this.bundleLoadedKey = createBundleLoadedKey(this.persistenceKey); + // Rather than adding the storage observer during start(), we add the // storage observer during initialization. This ensures that we collect // events before other components populate their initial state (during their @@ -710,6 +720,10 @@ export class WebStorageSharedClientState implements SharedClientState { this.persistOnlineState(onlineState); } + notifyBundleLoaded(): void { + this.persistBundleLoadedState(); + } + shutdown(): void { if (this.started) { this.window.removeEventListener('storage', this.storageListener); @@ -817,6 +831,8 @@ export class WebStorageSharedClientState implements SharedClientState { if (sequenceNumber !== ListenSequence.INVALID) { this.sequenceNumberHandler!(sequenceNumber); } + } else if (storageEvent.key === this.bundleLoadedKey) { + return this.syncEngine!.synchronizeWithChangedDocuments(); } }); } @@ -882,6 +898,10 @@ export class WebStorageSharedClientState implements SharedClientState { this.setItem(targetKey, targetMetadata.toWebStorageJSON()); } + private persistBundleLoadedState(): void { + this.setItem(this.bundleLoadedKey, 'value-not-used'); + } + /** * Parses a client state key in WebStorage. Returns null if the key does not * match the expected key format. @@ -1128,4 +1148,8 @@ export class MemorySharedClientState implements SharedClientState { shutdown(): void {} writeSequenceNumber(sequenceNumber: ListenSequenceNumber): void {} + + notifyBundleLoaded(): void { + // No op. + } } diff --git a/packages/firestore/src/local/shared_client_state_schema.ts b/packages/firestore/src/local/shared_client_state_schema.ts index 01665e91cfa..2f8035fce4e 100644 --- a/packages/firestore/src/local/shared_client_state_schema.ts +++ b/packages/firestore/src/local/shared_client_state_schema.ts @@ -115,6 +115,15 @@ export function createWebStorageOnlineStateKey(persistenceKey: string): string { return `${ONLINE_STATE_KEY_PREFIX}_${persistenceKey}`; } +// The WebStorage prefix that plays as a event to indicate the remote documents +// might have changed due to some secondary tabs loading a bundle. +// format of the key is: +// firestore_bundle_loaded_ +export const BUNDLE_LOADED_KEY_PREFIX = 'firestore_bundle_loaded'; +export function createBundleLoadedKey(persistenceKey: string): string { + return `${BUNDLE_LOADED_KEY_PREFIX}_${persistenceKey}`; +} + /** * The JSON representation of the system's online state, as written by the * primary client. diff --git a/packages/firestore/src/local/shared_client_state_syncer.ts b/packages/firestore/src/local/shared_client_state_syncer.ts index 29736807277..898c8c78b6d 100644 --- a/packages/firestore/src/local/shared_client_state_syncer.ts +++ b/packages/firestore/src/local/shared_client_state_syncer.ts @@ -49,4 +49,10 @@ export interface SharedClientStateSyncer { /** Returns the IDs of the clients that are currently active. */ getActiveClients(): Promise; + + /** + * Retrieves newly changed documents from remote document cache and raises + * snapshots if needed. + */ + synchronizeWithChangedDocuments(): Promise; } diff --git a/packages/firestore/src/platform/browser/byte_stream_reader.ts b/packages/firestore/src/platform/browser/byte_stream_reader.ts new file mode 100644 index 00000000000..65875d4b225 --- /dev/null +++ b/packages/firestore/src/platform/browser/byte_stream_reader.ts @@ -0,0 +1,40 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BundleSource } from '../../util/bundle_reader'; +import { toByteStreamReaderHelper } from '../../util/byte_stream'; + +/** + * On web, a `ReadableStream` is wrapped around by a `ByteStreamReader`. + */ +export function toByteStreamReader( + source: BundleSource, + bytesPerRead: number +): ReadableStreamReader { + if (source instanceof Uint8Array) { + return toByteStreamReaderHelper(source, bytesPerRead); + } + if (source instanceof ArrayBuffer) { + return toByteStreamReaderHelper(new Uint8Array(source), bytesPerRead); + } + if (source instanceof ReadableStream) { + return source.getReader(); + } + throw new Error( + 'Source of `toByteStreamReader` has to be a ArrayBuffer or ReadableStream' + ); +} diff --git a/packages/firestore/src/platform/browser/serializer.ts b/packages/firestore/src/platform/browser/serializer.ts index 5e009d89f60..722f40e605f 100644 --- a/packages/firestore/src/platform/browser/serializer.ts +++ b/packages/firestore/src/platform/browser/serializer.ts @@ -22,3 +22,17 @@ import { JsonProtoSerializer } from '../../remote/serializer'; export function newSerializer(databaseId: DatabaseId): JsonProtoSerializer { return new JsonProtoSerializer(databaseId, /* useProto3Json= */ true); } + +/** + * An instance of the Platform's 'TextEncoder' implementation. + */ +export function newTextEncoder(): TextEncoder { + return new TextEncoder(); +} + +/** + * An instance of the Platform's 'TextDecoder' implementation. + */ +export function newTextDecoder(): TextDecoder { + return new TextDecoder('utf-8'); +} diff --git a/packages/firestore/src/platform/browser_lite/byte_stream_reader.ts b/packages/firestore/src/platform/browser_lite/byte_stream_reader.ts new file mode 100644 index 00000000000..235c3e70616 --- /dev/null +++ b/packages/firestore/src/platform/browser_lite/byte_stream_reader.ts @@ -0,0 +1,18 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from '../browser/byte_stream_reader'; diff --git a/packages/firestore/src/platform/browser_lite/serializer.ts b/packages/firestore/src/platform/browser_lite/serializer.ts index ed490bf16d7..5730c508f22 100644 --- a/packages/firestore/src/platform/browser_lite/serializer.ts +++ b/packages/firestore/src/platform/browser_lite/serializer.ts @@ -15,9 +15,4 @@ * limitations under the License. */ -import { JsonProtoSerializer } from '../../remote/serializer'; -import { DatabaseId } from '../../core/database_info'; - -export function newSerializer(databaseId: DatabaseId): JsonProtoSerializer { - return new JsonProtoSerializer(databaseId, /* useProto3Json= */ true); -} +export * from '../browser/serializer'; diff --git a/packages/firestore/src/platform/byte_stream_reader.ts b/packages/firestore/src/platform/byte_stream_reader.ts new file mode 100644 index 00000000000..49aac60e9ff --- /dev/null +++ b/packages/firestore/src/platform/byte_stream_reader.ts @@ -0,0 +1,36 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BundleSource } from '../util/bundle_reader'; +import * as node from './node/byte_stream_reader'; +import * as rn from './rn/byte_stream_reader'; +import * as browser from './browser/byte_stream_reader'; +import { isNode, isReactNative } from '@firebase/util'; +import { DEFAULT_BYTES_PER_READ } from '../util/byte_stream'; + +export function toByteStreamReader( + source: BundleSource, + bytesPerRead: number = DEFAULT_BYTES_PER_READ +): ReadableStreamReader { + if (isNode()) { + return node.toByteStreamReader(source, bytesPerRead); + } else if (isReactNative()) { + return rn.toByteStreamReader(source, bytesPerRead); + } else { + return browser.toByteStreamReader(source, bytesPerRead); + } +} diff --git a/packages/firestore/src/platform/node/byte_stream_reader.ts b/packages/firestore/src/platform/node/byte_stream_reader.ts new file mode 100644 index 00000000000..9add9ce3cc9 --- /dev/null +++ b/packages/firestore/src/platform/node/byte_stream_reader.ts @@ -0,0 +1,39 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BundleSource } from '../../util/bundle_reader'; +import { toByteStreamReaderHelper } from '../../util/byte_stream'; +import { Code, FirestoreError } from '../../util/error'; +import { valueDescription } from '../../util/input_validation'; + +/** + * On Node, only supported data source is a `Uint8Array` for now. + */ +export function toByteStreamReader( + source: BundleSource, + bytesPerRead: number +): ReadableStreamReader { + if (!(source instanceof Uint8Array)) { + throw new FirestoreError( + Code.INVALID_ARGUMENT, + `NodePlatform.toByteStreamReader expects source to be Uint8Array, got ${valueDescription( + source + )}` + ); + } + return toByteStreamReaderHelper(source, bytesPerRead); +} diff --git a/packages/firestore/src/platform/node/serializer.ts b/packages/firestore/src/platform/node/serializer.ts index bbc9db1e516..ac26afd0ee7 100644 --- a/packages/firestore/src/platform/node/serializer.ts +++ b/packages/firestore/src/platform/node/serializer.ts @@ -18,7 +18,22 @@ /** Return the Platform-specific serializer monitor. */ import { JsonProtoSerializer } from '../../remote/serializer'; import { DatabaseId } from '../../core/database_info'; +import { TextDecoder, TextEncoder } from 'util'; export function newSerializer(databaseId: DatabaseId): JsonProtoSerializer { return new JsonProtoSerializer(databaseId, /* useProto3Json= */ false); } + +/** + * An instance of the Platform's 'TextEncoder' implementation. + */ +export function newTextEncoder(): TextEncoder { + return new TextEncoder(); +} + +/** + * An instance of the Platform's 'TextDecoder' implementation. + */ +export function newTextDecoder(): TextDecoder { + return new TextDecoder('utf-8'); +} diff --git a/packages/firestore/src/platform/node_lite/byte_stream_reader.ts b/packages/firestore/src/platform/node_lite/byte_stream_reader.ts new file mode 100644 index 00000000000..576537f10b6 --- /dev/null +++ b/packages/firestore/src/platform/node_lite/byte_stream_reader.ts @@ -0,0 +1,18 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from '../node/byte_stream_reader'; diff --git a/packages/firestore/src/platform/node_lite/serializer.ts b/packages/firestore/src/platform/node_lite/serializer.ts index c53bf2cbd7e..3f2d664b97c 100644 --- a/packages/firestore/src/platform/node_lite/serializer.ts +++ b/packages/firestore/src/platform/node_lite/serializer.ts @@ -15,10 +15,4 @@ * limitations under the License. */ -/** Return the Platform-specific serializer monitor. */ -import { JsonProtoSerializer } from '../../remote/serializer'; -import { DatabaseId } from '../../core/database_info'; - -export function newSerializer(databaseId: DatabaseId): JsonProtoSerializer { - return new JsonProtoSerializer(databaseId, /* useProto3Json= */ true); -} +export * from '../node/serializer'; diff --git a/packages/firestore/src/platform/rn/byte_stream_reader.ts b/packages/firestore/src/platform/rn/byte_stream_reader.ts new file mode 100644 index 00000000000..709509c8a4e --- /dev/null +++ b/packages/firestore/src/platform/rn/byte_stream_reader.ts @@ -0,0 +1,18 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { toByteStreamReader } from '../browser/byte_stream_reader'; diff --git a/packages/firestore/src/platform/rn/serializer.ts b/packages/firestore/src/platform/rn/serializer.ts index c5ab7bf2bb5..2b168a0dffa 100644 --- a/packages/firestore/src/platform/rn/serializer.ts +++ b/packages/firestore/src/platform/rn/serializer.ts @@ -15,4 +15,8 @@ * limitations under the License. */ -export { newSerializer } from '../browser/serializer'; +export { + newSerializer, + newTextEncoder, + newTextDecoder +} from '../browser/serializer'; diff --git a/packages/firestore/src/platform/rn_lite/byte_stream_reader.ts b/packages/firestore/src/platform/rn_lite/byte_stream_reader.ts new file mode 100644 index 00000000000..5bd0e6a52ec --- /dev/null +++ b/packages/firestore/src/platform/rn_lite/byte_stream_reader.ts @@ -0,0 +1,18 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from '../rn/byte_stream_reader'; diff --git a/packages/firestore/src/platform/serializer.ts b/packages/firestore/src/platform/serializer.ts index c2e416abdc1..5ca086edb3b 100644 --- a/packages/firestore/src/platform/serializer.ts +++ b/packages/firestore/src/platform/serializer.ts @@ -14,8 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import { isNode, isReactNative } from '@firebase/util'; import { DatabaseId } from '../core/database_info'; import { JsonProtoSerializer } from '../remote/serializer'; +import * as node from './node/serializer'; +import * as rn from './rn/serializer'; +import * as browser from './browser/serializer'; // This file is only used under ts-node. // eslint-disable-next-line @typescript-eslint/no-require-imports @@ -24,3 +29,29 @@ const platform = require(`./${process.env.TEST_PLATFORM ?? 'node'}/serializer`); export function newSerializer(databaseId: DatabaseId): JsonProtoSerializer { return platform.newSerializer(databaseId); } + +/** + * An instance of the Platform's 'TextEncoder' implementation. + */ +export function newTextEncoder(): TextEncoder { + if (isNode()) { + return node.newTextEncoder(); + } else if (isReactNative()) { + return rn.newTextEncoder(); + } else { + return browser.newTextEncoder(); + } +} + +/** + * An instance of the Platform's 'TextDecoder' implementation. + */ +export function newTextDecoder(): TextDecoder { + if (isNode()) { + return node.newTextDecoder() as TextDecoder; + } else if (isReactNative()) { + return rn.newTextDecoder(); + } else { + return browser.newTextDecoder(); + } +} diff --git a/packages/firestore/src/protos/firestore/bundle.proto b/packages/firestore/src/protos/firestore/bundle.proto new file mode 100644 index 00000000000..ee7954e664c --- /dev/null +++ b/packages/firestore/src/protos/firestore/bundle.proto @@ -0,0 +1,121 @@ +// Copyright 2020 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// This file defines the format of Firestore bundle file/stream. It is not a part of the +// Firestore API, only a specification used by Server and Client SDK to write and read +// bundles. + +syntax = "proto3"; + +package firestore; + +import "google/firestore/v1/document.proto"; +import "google/firestore/v1/query.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Firestore.Proto"; +option go_package = "google.golang.org/genproto/firestore/proto;firestore"; +option java_multiple_files = true; +option java_outer_classname = "BundleProto"; +option java_package = "com.google.firestore.proto"; +option objc_class_prefix = "FSTPB"; +option php_namespace = "Firestore\\Proto"; + +// Describes a query saved in the bundle. +message BundledQuery { + // The parent resource name. + string parent = 1; + + // The query to run. + oneof query_type { + // A structured query. + google.firestore.v1.StructuredQuery structured_query = 2; + } + + // If the query is a limit query, should the limit be applied to the beginning or + // the end of results. + enum LimitType { + FIRST = 0; + LAST = 1; + } + LimitType limit_type = 3; +} + +// A Query associated with a name, created as part of the bundle file, and can be read +// by client SDKs once the bundle containing them is loaded. +message NamedQuery { + // Name of the query, such that client can use the name to load this query + // from bundle, and resume from when the query results are materialized + // into this bundle. + string name = 1; + + // The query saved in the bundle. + BundledQuery bundled_query = 2; + + // The read time of the query, when it is used to build the bundle. This is useful to + // resume the query from the bundle, once it is loaded by client SDKs. + google.protobuf.Timestamp read_time = 3; +} + +// Metadata describing a Firestore document saved in the bundle. +message BundledDocumentMetadata { + // The document key of a bundled document. + string name = 1; + + // The snapshot version of the document data bundled. + google.protobuf.Timestamp read_time = 2; + + // Whether the document exists. + bool exists = 3; + + // The names of the queries in this bundle that this document matches to. + repeated string queries = 4; +} + +// Metadata describing the bundle file/stream. +message BundleMetadata { + // The ID of the bundle. + string id = 1; + + // Time at which the documents snapshot is taken for this bundle. + google.protobuf.Timestamp create_time = 2; + + // The schema version of the bundle. + uint32 version = 3; + + // The number of documents in the bundle. + uint32 total_documents = 4; + + // The size of the bundle in bytes, excluding this `BundleMetadata`. + uint64 total_bytes = 5; +} + +// A Firestore bundle is a length-prefixed stream of JSON representations of +// `BundleElement`. +// Only one `BundleMetadata` is expected, and it should be the first element. +// The named queries follow after `metadata`. If a document exists when the +// bundle is built, `document_metadata` is immediately followed by the +// `document`, otherwise `document_metadata` will present by itself. +message BundleElement { + oneof element_type { + BundleMetadata metadata = 1; + + NamedQuery named_query = 2; + + BundledDocumentMetadata document_metadata = 3; + + google.firestore.v1.Document document = 4; + } +} diff --git a/packages/firestore/src/protos/firestore_bundle_proto.ts b/packages/firestore/src/protos/firestore_bundle_proto.ts new file mode 100644 index 00000000000..304fe3e68c7 --- /dev/null +++ b/packages/firestore/src/protos/firestore_bundle_proto.ts @@ -0,0 +1,93 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as api from './firestore_proto_api'; + +/** Properties of a BundledQuery. */ +export interface BundledQuery { + /** BundledQuery parent */ + parent?: string | null; + + /** BundledQuery structuredQuery */ + structuredQuery?: api.StructuredQuery | null; + + /** BundledQuery limitType */ + limitType?: LimitType | null; +} + +/** LimitType enum. */ +export type LimitType = 'FIRST' | 'LAST'; + +/** Properties of a NamedQuery. */ +export interface NamedQuery { + /** NamedQuery name */ + name?: string | null; + + /** NamedQuery bundledQuery */ + bundledQuery?: BundledQuery | null; + + /** NamedQuery readTime */ + readTime?: api.Timestamp | null; +} + +/** Properties of a BundledDocumentMetadata. */ +export interface BundledDocumentMetadata { + /** BundledDocumentMetadata name */ + name?: string | null; + + /** BundledDocumentMetadata readTime */ + readTime?: api.Timestamp | null; + + /** BundledDocumentMetadata exists */ + exists?: boolean | null; + + /** The names of the queries in this bundle that this document matches to. */ + queries?: string[]; +} + +/** Properties of a BundleMetadata. */ +export interface BundleMetadata { + /** BundleMetadata id */ + id?: string | null; + + /** BundleMetadata createTime */ + createTime?: api.Timestamp | null; + + /** BundleMetadata version */ + version?: number | null; + + /** BundleMetadata totalDocuments */ + totalDocuments?: number | null; + + /** BundleMetadata totalBytes */ + totalBytes?: number | null; +} + +/** Properties of a BundleElement. */ +export interface BundleElement { + /** BundleElement metadata */ + metadata?: BundleMetadata | null; + + /** BundleElement namedQuery */ + namedQuery?: NamedQuery | null; + + /** BundleElement documentMetadata */ + documentMetadata?: BundledDocumentMetadata | null; + + /** BundleElement document */ + document?: api.Document | null; +} diff --git a/packages/firestore/src/protos/firestore_proto_api.d.ts b/packages/firestore/src/protos/firestore_proto_api.d.ts index 071b502f04d..6a62a636024 100644 --- a/packages/firestore/src/protos/firestore_proto_api.d.ts +++ b/packages/firestore/src/protos/firestore_proto_api.d.ts @@ -192,7 +192,7 @@ export declare namespace firestoreV1ApiClientInterfaces { interface Document { name?: string; fields?: ApiClientObjectMap; - createTime?: string; + createTime?: Timestamp; updateTime?: Timestamp; } interface DocumentChange { @@ -353,7 +353,7 @@ export declare namespace firestoreV1ApiClientInterfaces { query?: QueryTarget; documents?: DocumentsTarget; resumeToken?: string | Uint8Array; - readTime?: string; + readTime?: Timestamp; targetId?: number; once?: boolean; } diff --git a/packages/firestore/src/remote/serializer.ts b/packages/firestore/src/remote/serializer.ts index cf73a799b5b..b9a622bf8a7 100644 --- a/packages/firestore/src/remote/serializer.ts +++ b/packages/firestore/src/remote/serializer.ts @@ -27,6 +27,7 @@ import { newQueryForPath, Operator, OrderBy, + Query, queryToTarget } from '../core/query'; import { SnapshotVersion } from '../core/snapshot_version'; @@ -338,21 +339,26 @@ export function fromName( name: string ): DocumentKey { const resource = fromResourceName(name); - hardAssert( - resource.get(1) === serializer.databaseId.projectId, - 'Tried to deserialize key from different project: ' + - resource.get(1) + - ' vs ' + - serializer.databaseId.projectId - ); - hardAssert( - (!resource.get(3) && !serializer.databaseId.database) || - resource.get(3) === serializer.databaseId.database, - 'Tried to deserialize key from different database: ' + - resource.get(3) + - ' vs ' + - serializer.databaseId.database - ); + + if (resource.get(1) !== serializer.databaseId.projectId) { + throw new FirestoreError( + Code.INVALID_ARGUMENT, + 'Tried to deserialize key from different project: ' + + resource.get(1) + + ' vs ' + + serializer.databaseId.projectId + ); + } + + if (resource.get(3) !== serializer.databaseId.database) { + throw new FirestoreError( + Code.INVALID_ARGUMENT, + 'Tried to deserialize key from different database: ' + + resource.get(3) + + ' vs ' + + serializer.databaseId.database + ); + } return new DocumentKey(extractLocalPathFromResourceName(resource)); } @@ -887,7 +893,7 @@ export function toQueryTarget( return result; } -export function fromQueryTarget(target: ProtoQueryTarget): Target { +export function convertQueryTargetToQuery(target: ProtoQueryTarget): Query { let path = fromQueryPath(target.parent!); const query = target.structuredQuery!; @@ -931,20 +937,22 @@ export function fromQueryTarget(target: ProtoQueryTarget): Target { endAt = fromCursor(query.endAt); } - return queryToTarget( - newQuery( - path, - collectionGroup, - orderBy, - filterBy, - limit, - LimitType.First, - startAt, - endAt - ) + return newQuery( + path, + collectionGroup, + orderBy, + filterBy, + limit, + LimitType.First, + startAt, + endAt ); } +export function fromQueryTarget(target: ProtoQueryTarget): Target { + return queryToTarget(convertQueryTargetToQuery(target)); +} + export function toListenRequestLabels( serializer: JsonProtoSerializer, targetData: TargetData @@ -992,6 +1000,14 @@ export function toTarget( if (targetData.resumeToken.approximateByteSize() > 0) { result.resumeToken = toBytes(serializer, targetData.resumeToken); + } else if (targetData.snapshotVersion.compareTo(SnapshotVersion.min()) > 0) { + // TODO(wuandy): Consider removing above check because it is most likely true. + // Right now, many tests depend on this behaviour though (leaving min() out + // of serialization). + result.readTime = toTimestamp( + serializer, + targetData.snapshotVersion.toTimestamp() + ); } return result; diff --git a/packages/firestore/src/util/bundle_reader.ts b/packages/firestore/src/util/bundle_reader.ts new file mode 100644 index 00000000000..ae3badda147 --- /dev/null +++ b/packages/firestore/src/util/bundle_reader.ts @@ -0,0 +1,241 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + BundleElement, + BundleMetadata +} from '../protos/firestore_bundle_proto'; +import { Deferred } from './promise'; +import { debugAssert } from './assert'; +import { toByteStreamReader } from '../platform/byte_stream_reader'; +import { newTextDecoder } from '../platform/serializer'; +import { JsonProtoSerializer } from '../remote/serializer'; + +/** + * A complete element in the bundle stream, together with the byte length it + * occupies in the stream. + */ +export class SizedBundleElement { + constructor( + public readonly payload: BundleElement, + // How many bytes this element takes to store in the bundle. + public readonly byteLength: number + ) {} + + isBundleMetadata(): boolean { + return 'metadata' in this.payload; + } +} + +export type BundleSource = + | ReadableStream + | ArrayBuffer + | Uint8Array; + +/** + * When applicable, how many bytes to read from the underlying data source + * each time. + * + * Not applicable for ReadableStreams. + */ +const BYTES_PER_READ = 10240; + +/** + * A class representing a bundle. + * + * Takes a bundle stream or buffer, and presents abstractions to read bundled + * elements out of the underlying content. + */ +export class BundleReader { + /** Cached bundle metadata. */ + private metadata: Deferred = new Deferred(); + /** + * Internal buffer to hold bundle content, accumulating incomplete element + * content. + */ + private buffer: Uint8Array = new Uint8Array(); + /** The decoder used to parse binary data into strings. */ + private textDecoder: TextDecoder; + + static fromBundleSource( + source: BundleSource, + serializer: JsonProtoSerializer + ): BundleReader { + return new BundleReader( + toByteStreamReader(source, BYTES_PER_READ), + serializer + ); + } + + constructor( + /** The reader to read from underlying binary bundle data source. */ + private reader: ReadableStreamReader, + readonly serializer: JsonProtoSerializer + ) { + this.textDecoder = newTextDecoder(); + // Read the metadata (which is the first element). + this.nextElementImpl().then( + element => { + if (element && element.isBundleMetadata()) { + this.metadata.resolve(element.payload.metadata!); + } else { + this.metadata.reject( + new Error(`The first element of the bundle is not a metadata, it is + ${JSON.stringify(element?.payload)}`) + ); + } + }, + error => this.metadata.reject(error) + ); + } + + close(): Promise { + return this.reader.cancel(); + } + + /** + * Returns the metadata of the bundle. + */ + async getMetadata(): Promise { + return this.metadata.promise; + } + + /** + * Returns the next BundleElement (together with its byte size in the bundle) + * that has not been read from underlying ReadableStream. Returns null if we + * have reached the end of the stream. + */ + async nextElement(): Promise { + // Makes sure metadata is read before proceeding. + await this.getMetadata(); + return this.nextElementImpl(); + } + + /** + * Reads from the head of internal buffer, and pulling more data from + * underlying stream if a complete element cannot be found, until an + * element(including the prefixed length and the JSON string) is found. + * + * Once a complete element is read, it is dropped from internal buffer. + * + * Returns either the bundled element, or null if we have reached the end of + * the stream. + */ + private async nextElementImpl(): Promise { + const lengthBuffer = await this.readLength(); + if (lengthBuffer === null) { + return null; + } + + const lengthString = this.textDecoder.decode(lengthBuffer); + const length = Number(lengthString); + if (isNaN(length)) { + this.raiseError(`length string (${lengthString}) is not valid number`); + } + + const jsonString = await this.readJsonString(length); + + return new SizedBundleElement( + JSON.parse(jsonString), + lengthBuffer.length + length + ); + } + + /** First index of '{' from the underlying buffer. */ + private indexOfOpenBracket(): number { + return this.buffer.findIndex(v => v === '{'.charCodeAt(0)); + } + + /** + * Reads from the beginning of the internal buffer, until the first '{', and + * return the content. + * + * If reached end of the stream, returns a null. + */ + private async readLength(): Promise { + while (this.indexOfOpenBracket() < 0) { + const done = await this.pullMoreDataToBuffer(); + if (done) { + break; + } + } + + // Broke out of the loop because underlying stream is closed, and there + // happens to be no more data to process. + if (this.buffer.length === 0) { + return null; + } + + const position = this.indexOfOpenBracket(); + // Broke out of the loop because underlying stream is closed, but still + // cannot find an open bracket. + if (position < 0) { + this.raiseError( + 'Reached the end of bundle when a length string is expected.' + ); + } + + const result = this.buffer.slice(0, position); + // Update the internal buffer to drop the read length. + this.buffer = this.buffer.slice(position); + return result; + } + + /** + * Reads from a specified position from the internal buffer, for a specified + * number of bytes, pulling more data from the underlying stream if needed. + * + * Returns a string decoded from the read bytes. + */ + private async readJsonString(length: number): Promise { + while (this.buffer.length < length) { + const done = await this.pullMoreDataToBuffer(); + if (done) { + this.raiseError('Reached the end of bundle when more is expected.'); + } + } + + const result = this.textDecoder.decode(this.buffer.slice(0, length)); + // Update the internal buffer to drop the read json string. + this.buffer = this.buffer.slice(length); + return result; + } + + private raiseError(message: string): void { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + this.reader.cancel(); + throw new Error(`Invalid bundle format: ${message}`); + } + + /** + * Pulls more data from underlying stream to internal buffer. + * Returns a boolean indicating whether the stream is finished. + */ + private async pullMoreDataToBuffer(): Promise { + const result = await this.reader.read(); + if (!result.done) { + debugAssert(!!result.value, 'Read undefined when "done" is false.'); + const newBuffer = new Uint8Array( + this.buffer.length + result.value!.length + ); + newBuffer.set(this.buffer); + newBuffer.set(result.value!, this.buffer.length); + this.buffer = newBuffer; + } + return result.done; + } +} diff --git a/packages/firestore/src/util/byte_stream.ts b/packages/firestore/src/util/byte_stream.ts new file mode 100644 index 00000000000..a2723a82528 --- /dev/null +++ b/packages/firestore/src/util/byte_stream.ts @@ -0,0 +1,59 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { debugAssert } from './assert'; + +/** + * How many bytes to read each time when `ReadableStreamReader.read()` is + * called. Only applicable for byte streams that we control (e.g. those backed + * by an UInt8Array). + */ +export const DEFAULT_BYTES_PER_READ = 10240; + +/** + * Builds a `ByteStreamReader` from a UInt8Array. + * @param source The data source to use. + * @param bytesPerRead How many bytes each `read()` from the returned reader + * will read. + */ +export function toByteStreamReaderHelper( + source: Uint8Array, + bytesPerRead: number = DEFAULT_BYTES_PER_READ +): ReadableStreamReader { + debugAssert( + bytesPerRead > 0, + `toByteStreamReader expects positive bytesPerRead, but got ${bytesPerRead}` + ); + let readFrom = 0; + const reader: ReadableStreamReader = { + async read(): Promise> { + if (readFrom < source.byteLength) { + const result = { + value: source.slice(readFrom, readFrom + bytesPerRead), + done: false + }; + readFrom += bytesPerRead; + return result; + } + + return { value: undefined, done: true }; + }, + async cancel(): Promise {}, + releaseLock() {} + }; + return reader; +} diff --git a/packages/firestore/test/integration/api/bundle.test.ts b/packages/firestore/test/integration/api/bundle.test.ts new file mode 100644 index 00000000000..cfc507d7e62 --- /dev/null +++ b/packages/firestore/test/integration/api/bundle.test.ts @@ -0,0 +1,263 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// TODO(wuandy): Uncomment this file once prototype patch works for bundles + +// import * as firestore from '@firebase/firestore-types'; +// import { expect } from 'chai'; +// import { +// apiDescribe, +// toDataArray, +// withAlternateTestDb, +// withTestDb +// } from '../util/helpers'; +// import { EventsAccumulator } from '../util/events_accumulator'; +// import * as firebaseExport from '../util/firebase_export'; +// +// const loadBundle = firebaseExport.loadBundle; +// const namedQuery = firebaseExport.namedQuery; +// +// export const encoder = new TextEncoder(); +// +// function verifySuccessProgress(p: firestore.LoadBundleTaskProgress): void { +// expect(p.taskState).to.equal('Success'); +// expect(p.bytesLoaded).to.be.equal(p.totalBytes); +// expect(p.documentsLoaded).to.equal(p.totalDocuments); +// } +// +// function verifyInProgress( +// p: firestore.LoadBundleTaskProgress, +// expectedDocuments: number +// ): void { +// expect(p.taskState).to.equal('Running'); +// expect(p.bytesLoaded <= p.totalBytes).to.be.true; +// expect(p.documentsLoaded <= p.totalDocuments).to.be.true; +// expect(p.documentsLoaded).to.equal(expectedDocuments); +// } +// +// // This template is generated from bundleWithTestDocsAndQueries in '../util/internal_helpsers.ts', +// // and manually copied here. +// const BUNDLE_TEMPLATE = [ +// '{"metadata":{"id":"test-bundle","createTime":{"seconds":1001,"nanos":9999},"version":1,"totalDocuments":2,"totalBytes":1503}}', +// '{"namedQuery":{"name":"limit","readTime":{"seconds":1000,"nanos":9999},"bundledQuery":{"parent":"projects/{0}/databases/(default)/documents","structuredQuery":{"from":[{"collectionId":"coll-1"}],"orderBy":[{"field":{"fieldPath":"bar"},"direction":"DESCENDING"},{"field":{"fieldPath":"__name__"},"direction":"DESCENDING"}],"limit":{"value":1}},"limitType":"FIRST"}}}', +// '{"namedQuery":{"name":"limit-to-last","readTime":{"seconds":1000,"nanos":9999},"bundledQuery":{"parent":"projects/{0}/databases/(default)/documents","structuredQuery":{"from":[{"collectionId":"coll-1"}],"orderBy":[{"field":{"fieldPath":"bar"},"direction":"DESCENDING"},{"field":{"fieldPath":"__name__"},"direction":"DESCENDING"}],"limit":{"value":1}},"limitType":"LAST"}}}', +// '{"documentMetadata":{"name":"projects/{0}/databases/(default)/documents/coll-1/a","readTime":{"seconds":1000,"nanos":9999},"exists":true}}', +// '{"document":{"name":"projects/{0}/databases/(default)/documents/coll-1/a","createTime":{"seconds":1,"nanos":9},"updateTime":{"seconds":1,"nanos":9},"fields":{"k":{"stringValue":"a"},"bar":{"integerValue":1}}}}', +// '{"documentMetadata":{"name":"projects/{0}/databases/(default)/documents/coll-1/b","readTime":{"seconds":1000,"nanos":9999},"exists":true}}', +// '{"document":{"name":"projects/{0}/databases/(default)/documents/coll-1/b","createTime":{"seconds":1,"nanos":9},"updateTime":{"seconds":1,"nanos":9},"fields":{"k":{"stringValue":"b"},"bar":{"integerValue":2}}}}' +// ]; +// +// apiDescribe('Bundles', (persistence: boolean) => { +// function verifySnapEqualsTestDocs(snap: firestore.QuerySnapshot): void { +// expect(toDataArray(snap)).to.deep.equal([ +// { k: 'a', bar: 1 }, +// { k: 'b', bar: 2 } +// ]); +// } +// +// /** +// * Returns a valid bundle string from replacing project id in `BUNDLE_TEMPLATE` with the given +// * db project id (also recalculate length prefixes). +// */ +// function bundleString(db: firestore.FirebaseFirestore): string { +// const projectId: string = db.app.options.projectId; +// +// // Extract elements from BUNDLE_TEMPLATE and replace the project ID. +// const elements = BUNDLE_TEMPLATE.map(e => e.replace('{0}', projectId)); +// +// // Recalculating length prefixes for elements that are not BundleMetadata. +// let bundleContent = ''; +// for (const element of elements.slice(1)) { +// const length = encoder.encode(element).byteLength; +// bundleContent += `${length}${element}`; +// } +// +// // Update BundleMetadata with new totalBytes. +// const totalBytes = encoder.encode(bundleContent).byteLength; +// const metadata = JSON.parse(elements[0]); +// metadata.metadata.totalBytes = totalBytes; +// const metadataContent = JSON.stringify(metadata); +// const metadataLength = encoder.encode(metadataContent).byteLength; +// return `${metadataLength}${metadataContent}${bundleContent}`; +// } +// +// it('load with documents only with on progress and promise interface', () => { +// return withTestDb(persistence, async db => { +// const progressEvents: firestore.LoadBundleTaskProgress[] = []; +// let completeCalled = false; +// const task: firestore.LoadBundleTask = loadBundle(db, bundleString(db)); +// task.onProgress( +// progress => { +// progressEvents.push(progress); +// }, +// undefined, +// () => { +// completeCalled = true; +// } +// ); +// await task; +// let fulfillProgress: firestore.LoadBundleTaskProgress; +// await task.then(progress => { +// fulfillProgress = progress; +// }); +// +// expect(completeCalled).to.be.true; +// expect(progressEvents.length).to.equal(4); +// verifyInProgress(progressEvents[0], 0); +// verifyInProgress(progressEvents[1], 1); +// verifyInProgress(progressEvents[2], 2); +// verifySuccessProgress(progressEvents[3]); +// expect(fulfillProgress!).to.deep.equal(progressEvents[3]); +// +// // Read from cache. These documents do not exist in backend, so they can +// // only be read from cache. +// let snap = await db.collection('coll-1').get({ source: 'cache' }); +// verifySnapEqualsTestDocs(snap); +// +// snap = await (await namedQuery(db, 'limit'))!.get({ +// source: 'cache' +// }); +// expect(toDataArray(snap)).to.deep.equal([{ k: 'b', bar: 2 }]); +// +// snap = await (await namedQuery(db, 'limit-to-last'))!.get({ +// source: 'cache' +// }); +// expect(toDataArray(snap)).to.deep.equal([{ k: 'a', bar: 1 }]); +// }); +// }); +// +// it('load with documents and queries with promise interface', () => { +// return withTestDb(persistence, async db => { +// const fulfillProgress: firestore.LoadBundleTaskProgress = await loadBundle( +// db, +// bundleString(db) +// ); +// +// verifySuccessProgress(fulfillProgress!); +// +// // Read from cache. These documents do not exist in backend, so they can +// // only be read from cache. +// const snap = await db.collection('coll-1').get({ source: 'cache' }); +// verifySnapEqualsTestDocs(snap); +// }); +// }); +// +// it('load for a second time skips', () => { +// return withTestDb(persistence, async db => { +// await loadBundle(db, bundleString(db)); +// +// let completeCalled = false; +// const progressEvents: firestore.LoadBundleTaskProgress[] = []; +// const task: firestore.LoadBundleTask = loadBundle( +// db, +// encoder.encode(bundleString(db)) +// ); +// task.onProgress( +// progress => { +// progressEvents.push(progress); +// }, +// error => {}, +// () => { +// completeCalled = true; +// } +// ); +// await task; +// +// expect(completeCalled).to.be.true; +// // No loading actually happened in the second `loadBundle` call only the +// // success progress is recorded. +// expect(progressEvents.length).to.equal(1); +// verifySuccessProgress(progressEvents[0]); +// +// // Read from cache. These documents do not exist in backend, so they can +// // only be read from cache. +// const snap = await db.collection('coll-1').get({ source: 'cache' }); +// verifySnapEqualsTestDocs(snap); +// }); +// }); +// +// it('load with documents already pulled from backend', () => { +// return withTestDb(persistence, async db => { +// await db.doc('coll-1/a').set({ k: 'a', bar: 0 }); +// await db.doc('coll-1/b').set({ k: 'b', bar: 0 }); +// +// const accumulator = new EventsAccumulator(); +// db.collection('coll-1').onSnapshot(accumulator.storeEvent); +// await accumulator.awaitEvent(); +// +// const progress = await loadBundle( +// db, +// // Testing passing in non-string bundles. +// encoder.encode(bundleString(db)) +// ); +// +// verifySuccessProgress(progress); +// // The test bundle is holding ancient documents, so no events are +// // generated as a result. The case where a bundle has newer doc than +// // cache can only be tested in spec tests. +// await accumulator.assertNoAdditionalEvents(); +// +// let snap = await (await namedQuery(db, 'limit'))!.get(); +// expect(toDataArray(snap)).to.deep.equal([{ k: 'b', bar: 0 }]); +// +// snap = await (await namedQuery(db, 'limit-to-last'))!.get(); +// expect(toDataArray(snap)).to.deep.equal([{ k: 'a', bar: 0 }]); +// }); +// }); +// +// it('loaded documents should not be GC-ed right away', () => { +// return withTestDb(persistence, async db => { +// const fulfillProgress: firestore.LoadBundleTaskProgress = await loadBundle( +// db, +// bundleString(db) +// ); +// +// verifySuccessProgress(fulfillProgress!); +// +// // Read a different collection, this will trigger GC. +// let snap = await db.collection('coll-other').get(); +// expect(snap.empty).to.be.true; +// +// // Read the loaded documents, expecting document in cache. With memory +// // GC, the documents would get GC-ed if we did not hold the document keys +// // in a "umbrella" target. See local_store.ts for details. +// snap = await db.collection('coll-1').get({ source: 'cache' }); +// verifySnapEqualsTestDocs(snap); +// }); +// }); +// +// it('load with documents from other projects fails', () => { +// return withTestDb(persistence, async db => { +// return withAlternateTestDb(persistence, async otherDb => { +// await expect(loadBundle(otherDb, bundleString(db))).to.be.rejectedWith( +// 'Tried to deserialize key from different project' +// ); +// +// // Verify otherDb still functions, despite loaded a problematic bundle. +// const finalProgress = await loadBundle(otherDb, bundleString(otherDb)); +// verifySuccessProgress(finalProgress); +// +// // Read from cache. These documents do not exist in backend, so they can +// // only be read from cache. +// const snap = await otherDb +// .collection('coll-1') +// .get({ source: 'cache' }); +// verifySnapEqualsTestDocs(snap); +// }); +// }); +// }); +// }); diff --git a/packages/firestore/test/integration/util/internal_helpers.ts b/packages/firestore/test/integration/util/internal_helpers.ts index afef7ba7204..10ec071d204 100644 --- a/packages/firestore/test/integration/util/internal_helpers.ts +++ b/packages/firestore/test/integration/util/internal_helpers.ts @@ -32,6 +32,9 @@ import { User } from '../../../src/auth/user'; import { DEFAULT_PROJECT_ID, DEFAULT_SETTINGS } from './settings'; import { newConnection } from '../../../src/platform/connection'; import { newSerializer } from '../../../src/platform/serializer'; +import { key } from '../../util/helpers'; +import { TestBundleBuilder } from '../../unit/util/bundle_data'; +import { collectionReference } from '../../util/api_helpers'; /** Helper to retrieve the AsyncQueue for a give FirebaseFirestore instance. */ export function asyncQueue(db: firestore.FirebaseFirestore): AsyncQueue { @@ -95,3 +98,60 @@ export function withMockCredentialProviderTestDb( } ); } + +/** + * Returns a testing bundle string for the given projectId. + * + * The function is not referenced by bundle.test.ts, instead the bundle string used there + * is generated by this function and copied over there. The reason is this function accesses + * SDK internals, which is not available in test:minified. + */ +// eslint-disable-next-line @typescript-eslint/no-unused-vars +function bundleWithTestDocsAndQueries( + projectId: string = 'test-project' +): string { + const testDocs: { [key: string]: firestore.DocumentData } = { + a: { k: { stringValue: 'a' }, bar: { integerValue: 1 } }, + b: { k: { stringValue: 'b' }, bar: { integerValue: 2 } } + }; + + const a = key('coll-1/a'); + const b = key('coll-1/b'); + const builder = new TestBundleBuilder(new DatabaseId(projectId)); + + builder.addNamedQuery( + 'limit', + { seconds: 1000, nanos: 9999 }, + (collectionReference('coll-1') + .orderBy('bar', 'desc') + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .limit(1) as any)._query + ); + builder.addNamedQuery( + 'limit-to-last', + { seconds: 1000, nanos: 9999 }, + (collectionReference('coll-1') + .orderBy('bar', 'desc') + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .limitToLast(1) as any)._query + ); + + builder.addDocumentMetadata(a, { seconds: 1000, nanos: 9999 }, true); + builder.addDocument( + a, + { seconds: 1, nanos: 9 }, + { seconds: 1, nanos: 9 }, + testDocs.a + ); + builder.addDocumentMetadata(b, { seconds: 1000, nanos: 9999 }, true); + builder.addDocument( + b, + { seconds: 1, nanos: 9 }, + { seconds: 1, nanos: 9 }, + testDocs.b + ); + + return builder + .build('test-bundle', { seconds: 1001, nanos: 9999 }) + .toString(); +} diff --git a/packages/firestore/test/unit/local/bundle_cache.test.ts b/packages/firestore/test/unit/local/bundle_cache.test.ts new file mode 100644 index 00000000000..dfc84ee6b90 --- /dev/null +++ b/packages/firestore/test/unit/local/bundle_cache.test.ts @@ -0,0 +1,225 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; +import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; +import { filter, orderBy, query } from '../../util/helpers'; +import { TestBundleCache } from './test_bundle_cache'; +import { SnapshotVersion } from '../../../src/core/snapshot_version'; +import { Timestamp } from '../../../src/api/timestamp'; +import { + LimitType, + newQueryForCollectionGroup, + Query, + queryEquals, + queryToTarget, + queryWithLimit +} from '../../../src/core/query'; +import { + clearTestPersistence, + JSON_SERIALIZER, + testIndexedDbPersistence, + testMemoryEagerPersistence +} from './persistence_test_helpers'; +import { NamedQuery } from '../../../src/core/bundle'; +import { toQueryTarget } from '../../../src/remote/serializer'; + +describe('MemoryBundleCache', () => { + let cache: TestBundleCache; + + beforeEach(async () => { + cache = await testMemoryEagerPersistence().then( + persistence => new TestBundleCache(persistence) + ); + }); + + genericBundleCacheTests(() => cache); +}); + +describe('IndexedDbBundleCache', () => { + if (!IndexedDbPersistence.isAvailable()) { + console.warn('No IndexedDB. Skipping IndexedDbBundleCache tests.'); + return; + } + + let cache: TestBundleCache; + let persistence: IndexedDbPersistence; + beforeEach(async () => { + persistence = await testIndexedDbPersistence(); + cache = new TestBundleCache(persistence); + }); + + afterEach(async () => { + await persistence.shutdown(); + await clearTestPersistence(); + }); + + genericBundleCacheTests(() => cache); +}); + +/** + * Defines the set of tests to run against both bundle cache implementations. + */ +function genericBundleCacheTests(cacheFn: () => TestBundleCache): void { + let cache: TestBundleCache; + + beforeEach(async () => { + cache = cacheFn(); + }); + + function verifyNamedQuery( + actual: NamedQuery, + expectedName: string, + expectedQuery: Query, + expectedReadSeconds: number, + expectedReadNanos: number + ): void { + expect(actual.name).to.equal(expectedName); + expect(queryEquals(actual.query, expectedQuery)).to.be.true; + expect( + actual.readTime.isEqual( + SnapshotVersion.fromTimestamp( + new Timestamp(expectedReadSeconds, expectedReadNanos) + ) + ) + ).to.be.true; + } + + it('returns undefined when bundle id is not found', async () => { + expect(await cache.getBundle('bundle-1')).to.be.undefined; + }); + + it('returns saved bundle', async () => { + await cache.saveBundleMetadata({ + id: 'bundle-1', + version: 1, + createTime: { seconds: 1, nanos: 9999 } + }); + expect(await cache.getBundle('bundle-1')).to.deep.equal({ + id: 'bundle-1', + version: 1, + createTime: SnapshotVersion.fromTimestamp(new Timestamp(1, 9999)) + }); + + // Overwrite + await cache.saveBundleMetadata({ + id: 'bundle-1', + version: 2, + createTime: { seconds: 2, nanos: 1111 } + }); + expect(await cache.getBundle('bundle-1')).to.deep.equal({ + id: 'bundle-1', + version: 2, + createTime: SnapshotVersion.fromTimestamp(new Timestamp(2, 1111)) + }); + }); + + it('returns undefined when query name is not found', async () => { + expect(await cache.getNamedQuery('query-1')).to.be.undefined; + }); + + it('returns saved collection queries', async () => { + const query1 = query( + 'collection', + filter('sort', '>=', 2), + orderBy('sort') + ); + const queryTarget = toQueryTarget(JSON_SERIALIZER, queryToTarget(query1)); + + await cache.setNamedQuery({ + name: 'query-1', + readTime: { seconds: 1, nanos: 9999 }, + bundledQuery: { + parent: queryTarget.parent, + structuredQuery: queryTarget.structuredQuery + } + }); + + const namedQuery = await cache.getNamedQuery('query-1'); + verifyNamedQuery(namedQuery!, 'query-1', query1, 1, 9999); + }); + + it('returns saved collection group queries', async () => { + const query = newQueryForCollectionGroup('collection'); + const queryTarget = toQueryTarget(JSON_SERIALIZER, queryToTarget(query)); + + await cache.setNamedQuery({ + name: 'query-1', + readTime: { seconds: 1, nanos: 9999 }, + bundledQuery: { + parent: queryTarget.parent, + structuredQuery: queryTarget.structuredQuery, + limitType: undefined + } + }); + + const namedQuery = await cache.getNamedQuery('query-1'); + verifyNamedQuery(namedQuery!, 'query-1', query, 1, 9999); + }); + + it('returns expected limit queries', async () => { + const query1 = queryWithLimit( + query('collection', orderBy('sort')), + 3, + LimitType.First + ); + const queryTarget = toQueryTarget(JSON_SERIALIZER, queryToTarget(query1)); + + await cache.setNamedQuery({ + name: 'query-1', + readTime: { seconds: 1, nanos: 9999 }, + bundledQuery: { + parent: queryTarget.parent, + structuredQuery: queryTarget.structuredQuery, + limitType: 'FIRST' + } + }); + + const namedQuery = await cache.getNamedQuery('query-1'); + verifyNamedQuery(namedQuery!, 'query-1', query1, 1, 9999); + }); + + it('returns expected limit to last queries', async () => { + const query1 = queryWithLimit( + query('collection', orderBy('sort')), + 3, + LimitType.Last + ); + // Simulating bundle building for limit-to-last queries from the server + // SDKs: they save the equivelent limit-to-first queries with a limitType + // value 'LAST'. Client SDKs should apply a withLimitToLast when they see + // limitType 'LAST' from bundles. + const limitQuery = queryWithLimit(query1, 3, LimitType.First); + const queryTarget = toQueryTarget( + JSON_SERIALIZER, + queryToTarget(limitQuery) + ); + + await cache.setNamedQuery({ + name: 'query-1', + readTime: { seconds: 1, nanos: 9999 }, + bundledQuery: { + parent: queryTarget.parent, + structuredQuery: queryTarget.structuredQuery, + limitType: 'LAST' + } + }); + + const namedQuery = await cache.getNamedQuery('query-1'); + verifyNamedQuery(namedQuery!, 'query-1', query1, 1, 9999); + }); +} diff --git a/packages/firestore/test/unit/local/local_store.test.ts b/packages/firestore/test/unit/local/local_store.test.ts index 42b4c9d8004..a065fbc1c52 100644 --- a/packages/firestore/test/unit/local/local_store.test.ts +++ b/packages/firestore/test/unit/local/local_store.test.ts @@ -21,17 +21,28 @@ import { expect } from 'chai'; import { FieldValue } from '../../../src/compat/field_value'; import { Timestamp } from '../../../src/api/timestamp'; import { User } from '../../../src/auth/user'; -import { Query, queryToTarget } from '../../../src/core/query'; +import { + LimitType, + Query, + queryEquals, + queryToTarget, + queryWithLimit +} from '../../../src/core/query'; import { Target } from '../../../src/core/target'; import { BatchId, TargetId } from '../../../src/core/types'; import { SnapshotVersion } from '../../../src/core/snapshot_version'; import { IndexFreeQueryEngine } from '../../../src/local/index_free_query_engine'; import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; import { + applyBundleDocuments, + getNamedQuery, + hasNewerBundle, applyRemoteEventToLocalCache, LocalStore, LocalWriteResult, newLocalStore, + saveBundle, + saveNamedQuery, synchronizeLastDocumentChangeReadTime, notifyLocalViewChanges, acknowledgeBatch, @@ -48,6 +59,7 @@ import { LocalViewChanges } from '../../../src/local/local_view_changes'; import { Persistence } from '../../../src/local/persistence'; import { SimpleQueryEngine } from '../../../src/local/simple_query_engine'; import { + DocumentKeySet, documentKeySet, MaybeDocumentMap } from '../../../src/model/collections'; @@ -71,6 +83,8 @@ import { import { debugAssert } from '../../../src/util/assert'; import { addEqualityMatcher } from '../../util/equality_matcher'; import { + bundledDocuments, + bundleMetadata, byteStringFromString, deletedDoc, deleteMutation, @@ -82,10 +96,14 @@ import { key, localViewChanges, mapAsArray, + namedQuery, noChangeEvent, + orderBy, patchMutation, query, setMutation, + TestBundledDocuments, + TestNamedQuery, TestSnapshotVersion, transformMutation, unknownDoc, @@ -94,7 +112,10 @@ import { import { CountingQueryEngine, QueryEngineType } from './counting_query_engine'; import * as persistenceHelpers from './persistence_test_helpers'; +import { JSON_SERIALIZER } from './persistence_test_helpers'; import { ByteString } from '../../../src/util/byte_string'; +import { BundledDocuments, NamedQuery } from '../../../src/core/bundle'; +import * as bundleProto from '../../../src/protos/firestore_bundle_proto'; export interface LocalStoreComponents { queryEngine: CountingQueryEngine; @@ -123,7 +144,13 @@ class LocalStoreTester { } after( - op: Mutation | Mutation[] | RemoteEvent | LocalViewChanges + op: + | Mutation + | Mutation[] + | RemoteEvent + | LocalViewChanges + | TestBundledDocuments + | TestNamedQuery ): LocalStoreTester { if (op instanceof Mutation) { return this.afterMutations([op]); @@ -131,8 +158,12 @@ class LocalStoreTester { return this.afterMutations(op); } else if (op instanceof LocalViewChanges) { return this.afterViewChanges(op); - } else { + } else if (op instanceof RemoteEvent) { return this.afterRemoteEvent(op); + } else if (op instanceof TestBundledDocuments) { + return this.afterBundleDocuments(op.documents, op.bundleName); + } else { + return this.afterNamedQuery(op); } } @@ -161,6 +192,35 @@ class LocalStoreTester { return this; } + afterBundleDocuments( + documents: BundledDocuments, + bundleName?: string + ): LocalStoreTester { + this.prepareNextStep(); + + this.promiseChain = this.promiseChain + .then(() => + applyBundleDocuments(this.localStore, documents, bundleName || '') + ) + .then(result => { + this.lastChanges = result; + }); + return this; + } + + afterNamedQuery(testQuery: TestNamedQuery): LocalStoreTester { + this.prepareNextStep(); + + this.promiseChain = this.promiseChain.then(() => + saveNamedQuery( + this.localStore, + testQuery.namedQuery, + testQuery.matchingDocuments + ) + ); + return this; + } + afterViewChanges(viewChanges: LocalViewChanges): LocalStoreTester { this.prepareNextStep(); @@ -389,6 +449,60 @@ class LocalStoreTester { return this; } + toHaveQueryDocumentMapping( + persistence: Persistence, + targetId: TargetId, + expectedKeys: DocumentKeySet + ): LocalStoreTester { + this.promiseChain = this.promiseChain.then(() => { + return persistence.runTransaction( + 'toHaveQueryDocumentMapping', + 'readonly', + transaction => { + return persistence + .getTargetCache() + .getMatchingKeysForTargetId(transaction, targetId) + .next(matchedKeys => { + expect(matchedKeys.isEqual(expectedKeys)).to.be.true; + }); + } + ); + }); + + return this; + } + + toHaveNewerBundle( + metadata: bundleProto.BundleMetadata, + expected: boolean + ): LocalStoreTester { + this.promiseChain = this.promiseChain.then(() => { + return hasNewerBundle(this.localStore, metadata).then(actual => { + expect(actual).to.equal(expected); + }); + }); + return this; + } + + toHaveNamedQuery(namedQuery: NamedQuery): LocalStoreTester { + this.promiseChain = this.promiseChain.then(() => { + return getNamedQuery(this.localStore, namedQuery.name).then(actual => { + expect(actual).to.exist; + expect(actual!.name).to.equal(namedQuery.name); + expect(namedQuery.readTime.isEqual(actual!.readTime)).to.be.true; + expect(queryEquals(actual!.query, namedQuery.query)).to.be.true; + }); + }); + return this; + } + + afterSavingBundle(metadata: bundleProto.BundleMetadata): LocalStoreTester { + this.promiseChain = this.promiseChain.then(() => + saveBundle(this.localStore, metadata) + ); + return this; + } + finish(): Promise { return this.promiseChain; } @@ -406,7 +520,8 @@ describe('LocalStore w/ Memory Persistence (SimpleQueryEngine)', () => { const localStore = newLocalStore( persistence, queryEngine, - User.UNAUTHENTICATED + User.UNAUTHENTICATED, + JSON_SERIALIZER ); return { queryEngine, persistence, localStore }; } @@ -424,7 +539,8 @@ describe('LocalStore w/ Memory Persistence (IndexFreeQueryEngine)', () => { const localStore = newLocalStore( persistence, queryEngine, - User.UNAUTHENTICATED + User.UNAUTHENTICATED, + JSON_SERIALIZER ); return { queryEngine, persistence, localStore }; } @@ -450,7 +566,8 @@ describe('LocalStore w/ IndexedDB Persistence (SimpleQueryEngine)', () => { const localStore = newLocalStore( persistence, queryEngine, - User.UNAUTHENTICATED + User.UNAUTHENTICATED, + JSON_SERIALIZER ); await synchronizeLastDocumentChangeReadTime(localStore); return { queryEngine, persistence, localStore }; @@ -477,7 +594,8 @@ describe('LocalStore w/ IndexedDB Persistence (IndexFreeQueryEngine)', () => { const localStore = newLocalStore( persistence, queryEngine, - User.UNAUTHENTICATED + User.UNAUTHENTICATED, + JSON_SERIALIZER ); await synchronizeLastDocumentChangeReadTime(localStore); return { queryEngine, persistence, localStore }; @@ -1497,6 +1615,261 @@ function genericLocalStoreTests( .finish(); }); + it('handles saving bundled documents', () => { + return expectLocalStore() + .after( + bundledDocuments([ + doc('foo/bar', 1, { sum: 1337 }), + deletedDoc('foo/bar1', 1) + ]) + ) + .toReturnChanged( + doc('foo/bar', 1, { sum: 1337 }), + deletedDoc('foo/bar1', 1) + ) + .toContain(doc('foo/bar', 1, { sum: 1337 })) + .toContain(deletedDoc('foo/bar1', 1)) + .toHaveQueryDocumentMapping( + persistence, + /*targetId*/ 2, + /*expectedKeys*/ documentKeySet(key('foo/bar')) + ) + .finish(); + }); + + it('handles saving bundled documents with newer existing version', () => { + const query1 = query('foo'); + return expectLocalStore() + .afterAllocatingQuery(query1) + .toReturnTargetId(2) + .after(docAddedRemoteEvent(doc('foo/bar', 2, { sum: 1337 }), [2])) + .toContain(doc('foo/bar', 2, { sum: 1337 })) + .after( + bundledDocuments([ + doc('foo/bar', 1, { sum: 1336 }), + deletedDoc('foo/bar1', 1) + ]) + ) + .toReturnChanged(deletedDoc('foo/bar1', 1)) + .toContain(doc('foo/bar', 2, { sum: 1337 })) + .toContain(deletedDoc('foo/bar1', 1)) + .toHaveQueryDocumentMapping( + persistence, + /*targetId*/ 4, + /*expectedKeys*/ documentKeySet(key('foo/bar')) + ) + .finish(); + }); + + it('handles saving bundled documents with older existing version', () => { + const query1 = query('foo'); + return expectLocalStore() + .afterAllocatingQuery(query1) + .toReturnTargetId(2) + .after(docAddedRemoteEvent(doc('foo/bar', 1, { val: 'to-delete' }), [2])) + .toContain(doc('foo/bar', 1, { val: 'to-delete' })) + .after( + bundledDocuments([ + doc('foo/new', 1, { sum: 1336 }), + deletedDoc('foo/bar', 2) + ]) + ) + .toReturnChanged( + doc('foo/new', 1, { sum: 1336 }), + deletedDoc('foo/bar', 2) + ) + .toContain(doc('foo/new', 1, { sum: 1336 })) + .toContain(deletedDoc('foo/bar', 2)) + .toHaveQueryDocumentMapping( + persistence, + /*targetId*/ 4, + /*expectedKeys*/ documentKeySet(key('foo/new')) + ) + .finish(); + }); + + it('handles saving bundled documents with same existing version should not overwrite', () => { + const query1 = query('foo'); + return expectLocalStore() + .afterAllocatingQuery(query1) + .toReturnTargetId(2) + .after(docAddedRemoteEvent(doc('foo/bar', 1, { val: 'old' }), [2])) + .toContain(doc('foo/bar', 1, { val: 'old' })) + .after(bundledDocuments([doc('foo/bar', 1, { val: 'new' })])) + .toReturnChanged() + .toContain(doc('foo/bar', 1, { val: 'old' })) + .toHaveQueryDocumentMapping( + persistence, + /*targetId*/ 4, + /*expectedKeys*/ documentKeySet(key('foo/bar')) + ) + .finish(); + }); + + it('handles MergeMutation with Transform -> BundledDocuments', () => { + const query1 = query('foo'); + return expectLocalStore() + .afterAllocatingQuery(query1) + .toReturnTargetId(2) + .afterMutations([ + patchMutation('foo/bar', {}, Precondition.none()), + transformMutation('foo/bar', { sum: FieldValue.increment(1) }) + ]) + .toReturnChanged( + doc('foo/bar', 0, { sum: 1 }, { hasLocalMutations: true }) + ) + .toContain(doc('foo/bar', 0, { sum: 1 }, { hasLocalMutations: true })) + .after(bundledDocuments([doc('foo/bar', 1, { sum: 1337 })])) + .toReturnChanged( + doc('foo/bar', 1, { sum: 1 }, { hasLocalMutations: true }) + ) + .toContain(doc('foo/bar', 1, { sum: 1 }, { hasLocalMutations: true })) + .toHaveQueryDocumentMapping( + persistence, + /*targetId*/ 4, + /*expectedKeys*/ documentKeySet(key('foo/bar')) + ) + .finish(); + }); + + it('handles PatchMutation with Transform -> BundledDocuments', () => { + // Note: see comments in `handles PatchMutation with Transform -> RemoteEvent`. + // The behavior for this and remote event is the same. + + const query1 = query('foo'); + return expectLocalStore() + .afterAllocatingQuery(query1) + .toReturnTargetId(2) + .afterMutations([ + patchMutation('foo/bar', {}), + transformMutation('foo/bar', { sum: FieldValue.increment(1) }) + ]) + .toReturnChanged(deletedDoc('foo/bar', 0)) + .toNotContain('foo/bar') + .after(bundledDocuments([doc('foo/bar', 1, { sum: 1337 })])) + .toReturnChanged( + doc('foo/bar', 1, { sum: 1 }, { hasLocalMutations: true }) + ) + .toContain(doc('foo/bar', 1, { sum: 1 }, { hasLocalMutations: true })) + .toHaveQueryDocumentMapping( + persistence, + /*targetId*/ 4, + /*expectedKeys*/ documentKeySet(key('foo/bar')) + ) + .finish(); + }); + + it('handles saving and checking bundle metadata', () => { + return expectLocalStore() + .toHaveNewerBundle(bundleMetadata('test', 2), false) + .afterSavingBundle(bundleMetadata('test', 2)) + .toHaveNewerBundle(bundleMetadata('test', 1), true) + .finish(); + }); + + it('handles saving and loading named queries', async () => { + return expectLocalStore() + .after( + namedQuery( + 'testQueryName', + query('coll'), + /* limitType */ 'FIRST', + SnapshotVersion.min() + ) + ) + .toHaveNamedQuery({ + name: 'testQueryName', + query: query('coll'), + readTime: SnapshotVersion.min() + }) + .finish(); + }); + + it('loading named queries allocates targets and updates target document mapping', async () => { + const expectedQueryDocumentMap = new Map([ + ['query-1', documentKeySet(key('foo1/bar'))], + ['query-2', documentKeySet(key('foo2/bar'))] + ]); + const version1 = SnapshotVersion.fromTimestamp(Timestamp.fromMillis(10000)); + const version2 = SnapshotVersion.fromTimestamp(Timestamp.fromMillis(20000)); + + return expectLocalStore() + .after( + bundledDocuments( + [doc('foo1/bar', 1, { sum: 1337 }), doc('foo2/bar', 2, { sum: 42 })], + [['query-1'], ['query-2']] + ) + ) + .toReturnChanged( + doc('foo1/bar', 1, { sum: 1337 }), + doc('foo2/bar', 2, { sum: 42 }) + ) + .toContain(doc('foo1/bar', 1, { sum: 1337 })) + .toContain(doc('foo2/bar', 2, { sum: 42 })) + .after( + namedQuery( + 'query-1', + query('foo1'), + /* limitType */ 'FIRST', + version1, + expectedQueryDocumentMap.get('query-1') + ) + ) + .toHaveNamedQuery({ + name: 'query-1', + query: query('foo1'), + readTime: version1 + }) + .toHaveQueryDocumentMapping( + persistence, + /*targetId*/ 4, + /*expectedKeys*/ documentKeySet(key('foo1/bar')) + ) + .after( + namedQuery( + 'query-2', + query('foo2'), + /* limitType */ 'FIRST', + version2, + expectedQueryDocumentMap.get('query-2') + ) + ) + .toHaveNamedQuery({ + name: 'query-2', + query: query('foo2'), + readTime: version2 + }) + .toHaveQueryDocumentMapping( + persistence, + /*targetId*/ 6, + /*expectedKeys*/ documentKeySet(key('foo2/bar')) + ) + .finish(); + }); + + it('handles saving and loading limit to last queries', async () => { + const now = Timestamp.now(); + return expectLocalStore() + .after( + namedQuery( + 'testQueryName', + queryWithLimit(query('coll', orderBy('sort')), 5, LimitType.First), + /* limitType */ 'LAST', + SnapshotVersion.fromTimestamp(now) + ) + ) + .toHaveNamedQuery({ + name: 'testQueryName', + query: queryWithLimit( + query('coll', orderBy('sort')), + 5, + LimitType.Last + ), + readTime: SnapshotVersion.fromTimestamp(now) + }) + .finish(); + }); + it('computes highest unacknowledged batch id correctly', () => { return expectLocalStore() .toReturnHighestUnacknowledgeBatchId(BATCHID_UNKNOWN) diff --git a/packages/firestore/test/unit/local/persistence_test_helpers.ts b/packages/firestore/test/unit/local/persistence_test_helpers.ts index a5312f9628e..01bbc1a20ea 100644 --- a/packages/firestore/test/unit/local/persistence_test_helpers.ts +++ b/packages/firestore/test/unit/local/persistence_test_helpers.ts @@ -75,7 +75,7 @@ export const INDEXEDDB_TEST_DATABASE_NAME = indexedDbStoragePrefix(TEST_DATABASE_ID, TEST_PERSISTENCE_KEY) + MAIN_DATABASE; -const JSON_SERIALIZER = new JsonProtoSerializer( +export const JSON_SERIALIZER = new JsonProtoSerializer( TEST_DATABASE_ID, /* useProto3Json= */ true ); @@ -122,13 +122,16 @@ export async function testIndexedDbPersistence( /** Creates and starts a MemoryPersistence instance for testing. */ export async function testMemoryEagerPersistence(): Promise { - return new MemoryPersistence(MemoryEagerDelegate.factory); + return new MemoryPersistence(MemoryEagerDelegate.factory, JSON_SERIALIZER); } export async function testMemoryLruPersistence( params: LruParams = LruParams.DEFAULT ): Promise { - return new MemoryPersistence(p => new MemoryLruDelegate(p, params)); + return new MemoryPersistence( + p => new MemoryLruDelegate(p, params), + JSON_SERIALIZER + ); } /** Clears the persistence in tests */ diff --git a/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts b/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts index ec77f35695d..043e5905434 100644 --- a/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts +++ b/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts @@ -116,13 +116,4 @@ describe('RemoteDocumentChangeBuffer', () => { .catch(() => errors++) .then(() => expect(errors).to.equal(2)); }); - - it('cannot add documents with different read times', async () => { - // This test merely validates an assert that was added to the - // RemoteDocumentChangeBuffer to simplify our tracking of document - // read times. If we do need to track documents with different read - // times, this test should simply be removed. - buffer.addEntry(INITIAL_DOC, version(1)); - expect(() => buffer.addEntry(INITIAL_DOC, version(2))).to.throw(); - }); }); diff --git a/packages/firestore/test/unit/local/test_bundle_cache.ts b/packages/firestore/test/unit/local/test_bundle_cache.ts new file mode 100644 index 00000000000..2a0cdd99e8d --- /dev/null +++ b/packages/firestore/test/unit/local/test_bundle_cache.ts @@ -0,0 +1,73 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Persistence } from '../../../src/local/persistence'; +import { BundleCache } from '../../../src/local/bundle_cache'; +import { Bundle, NamedQuery } from '../../../src/core/bundle'; +import * as bundleProto from '../../../src/protos/firestore_bundle_proto'; + +/** + * A wrapper around a BundleCache that automatically creates a + * transaction around every operation to reduce test boilerplate. + */ +export class TestBundleCache { + private readonly cache: BundleCache; + + constructor(private readonly persistence: Persistence) { + this.cache = persistence.getBundleCache(); + } + + getBundle(bundleId: string): Promise { + return this.persistence.runTransaction( + 'getBundle', + 'readonly', + transaction => { + return this.cache.getBundleMetadata(transaction, bundleId); + } + ); + } + + saveBundleMetadata(metadata: bundleProto.BundleMetadata): Promise { + return this.persistence.runTransaction( + 'saveBundleMetadata', + 'readwrite', + transaction => { + return this.cache.saveBundleMetadata(transaction, metadata); + } + ); + } + + getNamedQuery(name: string): Promise { + return this.persistence.runTransaction( + 'getNamedQuery', + 'readonly', + transaction => { + return this.cache.getNamedQuery(transaction, name); + } + ); + } + + setNamedQuery(query: bundleProto.NamedQuery): Promise { + return this.persistence.runTransaction( + 'setNamedQuery', + 'readwrite', + transaction => { + return this.cache.saveNamedQuery(transaction, query); + } + ); + } +} diff --git a/packages/firestore/test/unit/local/web_storage_shared_client_state.test.ts b/packages/firestore/test/unit/local/web_storage_shared_client_state.test.ts index 5b3578649a9..dd70fe7b6ca 100644 --- a/packages/firestore/test/unit/local/web_storage_shared_client_state.test.ts +++ b/packages/firestore/test/unit/local/web_storage_shared_client_state.test.ts @@ -149,6 +149,8 @@ class TestSharedClientSyncer implements SharedClientStateSyncer { applyOnlineStateChange(onlineState: OnlineState): void { this.onlineState = onlineState; } + + async synchronizeWithChangedDocuments(): Promise {} } describe('WebStorageSharedClientState', () => { diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts new file mode 100644 index 00000000000..85399afd946 --- /dev/null +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -0,0 +1,472 @@ +/** + * @license + * Copyright 2017 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { newQueryForPath, Query } from '../../../src/core/query'; +import { + doc, + query, + filter, + TestSnapshotVersion, + version, + wrapObject +} from '../../util/helpers'; + +import { describeSpec, specTest } from './describe_spec'; +import { client, spec } from './spec_builder'; +import { TestBundleBuilder } from '../util/bundle_data'; +import { + JSON_SERIALIZER, + TEST_DATABASE_ID +} from '../local/persistence_test_helpers'; +import { DocumentKey } from '../../../src/model/document_key'; +import { toVersion } from '../../../src/remote/serializer'; +import { JsonObject } from '../../../src/model/object_value'; +import { LimitType } from '../../../src/protos/firestore_bundle_proto'; + +interface TestBundleDocument { + key: DocumentKey; + readTime: TestSnapshotVersion; + createTime?: TestSnapshotVersion; + updateTime?: TestSnapshotVersion; + content?: JsonObject; +} + +interface TestBundledQuery { + name: string; + readTime: TestSnapshotVersion; + query: Query; + limitType?: LimitType; +} + +function bundleWithDocumentAndQuery( + testDoc: TestBundleDocument, + testQuery?: TestBundledQuery +): string { + const builder = new TestBundleBuilder(TEST_DATABASE_ID); + + if (testQuery) { + builder.addNamedQuery( + testQuery.name, + toVersion(JSON_SERIALIZER, version(testQuery.readTime)), + testQuery.query + ); + } + + builder.addDocumentMetadata( + testDoc.key, + toVersion(JSON_SERIALIZER, version(testDoc.readTime)), + !!testDoc.createTime + ); + if (testDoc.createTime) { + builder.addDocument( + testDoc.key, + toVersion(JSON_SERIALIZER, version(testDoc.createTime)), + toVersion(JSON_SERIALIZER, version(testDoc.updateTime!)), + wrapObject(testDoc.content!).proto.mapValue.fields! + ); + } + return builder.build( + 'test-bundle', + toVersion(JSON_SERIALIZER, version(testDoc.readTime)) + ); +} + +describeSpec('Bundles:', ['no-ios', 'no-android'], () => { + specTest('Newer docs from bundles should overwrite cache', [], () => { + const query1 = query('collection'); + const docA = doc('collection/a', 1000, { value: 'a' }); + const docAChanged = doc('collection/a', 2999, { value: 'b' }); + + const bundleString = bundleWithDocumentAndQuery({ + key: docA.key, + readTime: 3000, + createTime: 1999, + updateTime: 2999, + content: { value: 'b' } + }); + + return ( + spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA) + .expectEvents(query1, { added: [docA] }) + // TODO(b/160876443): This currently raises snapshots with + // `fromCache=false` if users already listen to some queries and bundles + // has newer version. + .loadBundle(bundleString) + .expectEvents(query1, { modified: [docAChanged] }) + ); + }); + + specTest( + 'Newer deleted docs from bundles should delete cached docs', + [], + () => { + const query1 = query('collection'); + const docA = doc('collection/a', 1000, { value: 'a' }); + + const bundleString = bundleWithDocumentAndQuery({ + key: docA.key, + readTime: 3000 + }); + + return spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA) + .expectEvents(query1, { added: [docA] }) + .loadBundle(bundleString) + .expectEvents(query1, { removed: [docA] }); + } + ); + + specTest('Older deleted docs from bundles should do nothing', [], () => { + const query1 = query('collection'); + const docA = doc('collection/a', 1000, { value: 'a' }); + + const bundleString = bundleWithDocumentAndQuery({ + key: docA.key, + readTime: 999 + }); + + return ( + spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA) + .expectEvents(query1, { added: [docA] }) + // No events are expected here. + .loadBundle(bundleString) + ); + }); + + specTest( + 'Newer docs from bundles should raise snapshot only when Watch catches up with acknowledged writes', + [], + () => { + const query1 = query('collection'); + const docA = doc('collection/a', 250, { value: 'a' }); + + const bundleBeforeMutationAck = bundleWithDocumentAndQuery({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { value: 'b' } + }); + + const bundleAfterMutationAck = bundleWithDocumentAndQuery({ + key: docA.key, + readTime: 1001, + createTime: 250, + updateTime: 1001, + content: { value: 'fromBundle' } + }); + return ( + spec() + // TODO(b/160878667): Figure out what happens when memory eager GC is on + // a bundle is loaded. + .withGCEnabled(false) + .userListens(query1) + .watchAcksFull(query1, 250, docA) + .expectEvents(query1, { + added: [doc('collection/a', 250, { value: 'a' })] + }) + .userPatches('collection/a', { value: 'patched' }) + .expectEvents(query1, { + modified: [ + doc( + 'collection/a', + 250, + { value: 'patched' }, + { hasLocalMutations: true } + ) + ], + hasPendingWrites: true + }) + .writeAcks('collection/a', 1000) + // loading bundleBeforeMutationAck will not raise snapshots, because its + // snapshot version is older than the acknowledged mutation. + .loadBundle(bundleBeforeMutationAck) + // loading bundleAfterMutationAck will raise a snapshot, because it is after + // the acknowledged mutation. + .loadBundle(bundleAfterMutationAck) + .expectEvents(query1, { + modified: [doc('collection/a', 1001, { value: 'fromBundle' })] + }) + ); + } + ); + + specTest( + 'Newer docs from bundles should keep not raise snapshot if there are unacknowledged writes', + [], + () => { + const query1 = query('collection'); + const docA = doc('collection/a', 250, { value: 'a' }); + + const bundleString = bundleWithDocumentAndQuery({ + key: docA.key, + readTime: 1001, + createTime: 250, + updateTime: 1001, + content: { value: 'fromBundle' } + }); + + return ( + spec() + .withGCEnabled(false) + .userListens(query1) + .watchAcksFull(query1, 250, docA) + .expectEvents(query1, { + added: [doc('collection/a', 250, { value: 'a' })] + }) + .userPatches('collection/a', { value: 'patched' }) + .expectEvents(query1, { + modified: [ + doc( + 'collection/a', + 250, + { value: 'patched' }, + { hasLocalMutations: true } + ) + ], + hasPendingWrites: true + }) + // Loading the bundle will not raise snapshots, because the + // mutation has not been acknowledged. + .loadBundle(bundleString) + ); + } + ); + + specTest('Newer docs from bundles might lead to limbo doc', [], () => { + const query1 = query('collection'); + const docA = doc('collection/a', 1000, { value: 'a' }); + const bundleString1 = bundleWithDocumentAndQuery({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { value: 'b' } + }); + const limboQuery = newQueryForPath(docA.key.path); + + return ( + spec() + .withGCEnabled(false) + .userListens(query1) + .watchAcksFull(query1, 250) + // Backend tells is there is no such doc. + .expectEvents(query1, {}) + // Bundle tells otherwise, leads to limbo. + .loadBundle(bundleString1) + .expectLimboDocs(docA.key) + .expectEvents(query1, { + added: [doc('collection/a', 500, { value: 'b' })], + fromCache: true + }) + // .watchAcksFull(limboQuery, 1002, docA1) + .watchAcks(limboQuery) + .watchSends({ affects: [limboQuery] }) + .watchCurrents(limboQuery, 'resume-token-1002') + .watchSnapshots(1002) + .expectLimboDocs() + .expectEvents(query1, { + removed: [doc('collection/a', 500, { value: 'b' })], + fromCache: false + }) + ); + }); + + specTest('Bundles query can be resumed from same query.', [], () => { + const query1 = query('collection'); + const docA = doc('collection/a', 100, { key: 'a' }); + const bundleString1 = bundleWithDocumentAndQuery( + { + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { value: 'b' } + }, + { name: 'bundled-query', readTime: 400, query: query1 } + ); + + return spec() + .loadBundle(bundleString1) + .userListens(query1, { readTime: 400 }) + .expectEvents(query1, { + added: [doc('collection/a', 500, { value: 'b' })], + fromCache: true + }); + }); + + specTest( + 'Bundles query can be loaded and resumed from different tabs', + ['multi-client'], + () => { + const query1 = query('collection'); + const query2 = query('collection', filter('value', '==', 'c')); + const docA = doc('collection/a', 100, { value: 'a' }); + const bundleString1 = bundleWithDocumentAndQuery( + { + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { value: 'b' } + }, + { name: 'bundled-query', readTime: 400, query: query1 } + ); + + const bundleString2 = bundleWithDocumentAndQuery( + { + key: docA.key, + readTime: 600, + createTime: 250, + updateTime: 550, + content: { value: 'c' } + }, + { name: 'bundled-query', readTime: 560, query: query2 } + ); + + return ( + client(0) + .loadBundle(bundleString1) + // Read named query from loaded bundle by primary. + .client(1) + .userListens(query1, { readTime: 400 }) + .expectEvents(query1, { + added: [doc('collection/a', 500, { value: 'b' })], + fromCache: true + }) + // Loads a newer bundle. + .loadBundle(bundleString2) + .expectEvents(query1, { + modified: [doc('collection/a', 550, { value: 'c' })], + fromCache: true + }) + // Read named query from loaded bundle by secondary. + .client(0) + .expectListen(query1, { readTime: 400 }) + .expectActiveTargets({ query: query1, readTime: 400 }) + .userListens(query2, { readTime: 560 }) + .expectEvents(query2, { + added: [doc('collection/a', 550, { value: 'c' })], + fromCache: true + }) + ); + } + ); + + specTest( + 'Load from secondary clients and observe from primary', + ['multi-client'], + () => { + const query1 = query('collection'); + const docA = doc('collection/a', 250, { value: 'a' }); + const bundleString1 = bundleWithDocumentAndQuery({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { value: 'b' } + }); + + return client(0) + .userListens(query1) + .watchAcksFull(query1, 250, docA) + .expectEvents(query1, { + added: [docA] + }) + .client(1) + .loadBundle(bundleString1) + .client(0) + .expectEvents(query1, { + modified: [doc('collection/a', 500, { value: 'b' })] + }); + } + ); + + specTest( + 'Load and observe from same secondary client', + ['multi-client'], + () => { + const query1 = query('collection'); + const docA = doc('collection/a', 250, { value: 'a' }); + const bundleString = bundleWithDocumentAndQuery({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { value: 'b' } + }); + + return client(0) + .userListens(query1) + .watchAcksFull(query1, 250, docA) + .expectEvents(query1, { + added: [docA] + }) + .client(1) + .userListens(query1) + .expectEvents(query1, { + added: [docA] + }) + .loadBundle(bundleString) + .expectEvents(query1, { + modified: [doc('collection/a', 500, { value: 'b' })] + }); + } + ); + + specTest( + 'Load from primary client and observe from secondary', + ['multi-client'], + () => { + const query1 = query('collection'); + const docA = doc('collection/a', 250, { value: 'a' }); + const bundleString1 = bundleWithDocumentAndQuery({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { value: 'b' } + }); + + return client(0) + .userListens(query1) + .watchAcksFull(query1, 250, docA) + .expectEvents(query1, { + added: [docA] + }) + .client(1) + .userListens(query1) + .expectEvents(query1, { + added: [docA] + }) + .client(0) + .loadBundle(bundleString1) + .expectEvents(query1, { + modified: [doc('collection/a', 500, { value: 'b' })] + }) + .client(1) + .expectEvents(query1, { + modified: [doc('collection/a', 500, { value: 'b' })] + }); + } + ); +}); diff --git a/packages/firestore/test/unit/specs/existence_filter_spec.test.ts b/packages/firestore/test/unit/specs/existence_filter_spec.test.ts index 7ae233df6c1..30ea8fbe64f 100644 --- a/packages/firestore/test/unit/specs/existence_filter_spec.test.ts +++ b/packages/firestore/test/unit/specs/existence_filter_spec.test.ts @@ -74,7 +74,7 @@ describeSpec('Existence Filters:', [], () => { .watchAcksFull(query1, 1000, doc1) .expectEvents(query1, { added: [doc1] }) .userUnlistens(query1) - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [doc1], fromCache: true }) // The empty existence filter is ignored since Watch hasn't ACKed the // target diff --git a/packages/firestore/test/unit/specs/limbo_spec.test.ts b/packages/firestore/test/unit/specs/limbo_spec.test.ts index b2d628e1472..2500b43bd64 100644 --- a/packages/firestore/test/unit/specs/limbo_spec.test.ts +++ b/packages/firestore/test/unit/specs/limbo_spec.test.ts @@ -444,7 +444,7 @@ describeSpec('Limbo Documents:', [], () => { .expectEvents(query1, { fromCache: true }) .runTimer(TimerId.ClientMetadataRefresh) .expectPrimaryState(true) - .expectListen(query1, 'resume-token-1000000') + .expectListen(query1, { resumeToken: 'resume-token-1000000' }) .watchAcksFull(query1, 3 * 1e6) .expectLimboDocs(docB.key) .ackLimbo(4 * 1e6, deletedDocB) @@ -476,7 +476,7 @@ describeSpec('Limbo Documents:', [], () => { .expectLimboDocs(docB.key, docC.key) .client(1) .stealPrimaryLease() - .expectListen(query1, 'resume-token-1000000') + .expectListen(query1, { resumeToken: 'resume-token-1000000' }) .client(0) .runTimer(TimerId.ClientMetadataRefresh) .expectPrimaryState(false) @@ -489,7 +489,7 @@ describeSpec('Limbo Documents:', [], () => { .client(0) .expectEvents(query1, { removed: [docB], fromCache: true }) .stealPrimaryLease() - .expectListen(query1, 'resume-token-1000000') + .expectListen(query1, { resumeToken: 'resume-token-1000000' }) .watchAcksFull(query1, 5 * 1e6) .expectLimboDocs(docC.key) .ackLimbo(6 * 1e6, deletedDocC) @@ -545,7 +545,7 @@ describeSpec('Limbo Documents:', [], () => { // document `docBCommitted`, since we haven't received the resolved // document from Watch. Until we do, we return the version from cache // even though the backend told it does not match. - .userListens(originalQuery, 'resume-token-2000') + .userListens(originalQuery, { resumeToken: 'resume-token-2000' }) .expectEvents(originalQuery, { added: [docA, docBDirty], fromCache: true diff --git a/packages/firestore/test/unit/specs/limit_spec.test.ts b/packages/firestore/test/unit/specs/limit_spec.test.ts index 33ab16b90cb..16784697431 100644 --- a/packages/firestore/test/unit/specs/limit_spec.test.ts +++ b/packages/firestore/test/unit/specs/limit_spec.test.ts @@ -58,7 +58,7 @@ describeSpec('Limits:', [], () => { }) .userUnlistens(query1) .userSets('collection/c', { key: 'c' }) - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [doc1, doc2], fromCache: true @@ -203,7 +203,7 @@ describeSpec('Limits:', [], () => { .userUnlistens(limitQuery) .watchRemoves(limitQuery) .userSets('collection/a', { matches: false }) - .userListens(limitQuery, 'resume-token-1004') + .userListens(limitQuery, { resumeToken: 'resume-token-1004' }) .expectEvents(limitQuery, { added: [doc2, doc3], fromCache: true }); } ); @@ -238,7 +238,7 @@ describeSpec('Limits:', [], () => { .userUnlistens(limitQuery) .watchRemoves(limitQuery) .userSets('collection/a', { pos: 4 }) - .userListens(limitQuery, 'resume-token-1004') + .userListens(limitQuery, { resumeToken: 'resume-token-1004' }) .expectEvents(limitQuery, { added: [doc2, doc3], fromCache: true }); } ); @@ -273,11 +273,11 @@ describeSpec('Limits:', [], () => { .expectEvents(limitQuery, {}) .userUnlistens(limitQuery) .watchRemoves(limitQuery) - .userListens(fullQuery, 'resume-token-1003') + .userListens(fullQuery, { resumeToken: 'resume-token-1003' }) .expectEvents(fullQuery, { added: [doc1, doc2, doc3], fromCache: true }) .watchAcksFull(fullQuery, 1005, doc1Edited) .expectEvents(fullQuery, { modified: [doc1Edited] }) - .userListens(limitQuery, 'resume-token-1004') + .userListens(limitQuery, { resumeToken: 'resume-token-1004' }) .expectEvents(limitQuery, { added: [doc2, doc3], fromCache: true }); } ); @@ -332,7 +332,7 @@ describeSpec('Limits:', [], () => { // We listen to the limit query again. Note that we include // `firstDocument` in the local result since we did not resolve its // limbo state. - .userListens(limitQuery, 'resume-token-1001') + .userListens(limitQuery, { resumeToken: 'resume-token-1001' }) .expectEvents(limitQuery, { added: [firstDocument], fromCache: true }) .watchAcks(limitQuery) // Watch resumes the query from the provided resume token, but does @@ -400,7 +400,7 @@ describeSpec('Limits:', [], () => { .watchRemoves(fullQuery) // Re-issue the limit query and verify that we return `secondDocument` // from cache. - .userListens(limitQuery, 'resume-token-2001') + .userListens(limitQuery, { resumeToken: 'resume-token-2001' }) .expectEvents(limitQuery, { added: [secondDocument], fromCache: true diff --git a/packages/firestore/test/unit/specs/listen_spec.test.ts b/packages/firestore/test/unit/specs/listen_spec.test.ts index daa9c3f1217..ad0fa9ab201 100644 --- a/packages/firestore/test/unit/specs/listen_spec.test.ts +++ b/packages/firestore/test/unit/specs/listen_spec.test.ts @@ -278,7 +278,7 @@ describeSpec('Listens:', [], () => { // Remove and re-add listener. .userUnlistens(query1) .watchRemoves(query1) - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [docAv2], fromCache: true }) // watch sends old snapshot. .watchAcksFull(query1, 1000, docAv1) @@ -313,7 +313,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, { modified: [docAv2] }) // restart the client and re-listen. .restart() - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [docAv2], fromCache: true }) // watch sends old snapshot. .watchAcksFull(query1, 1000, docAv1) @@ -353,7 +353,7 @@ describeSpec('Listens:', [], () => { // us up to docAV2 since that's the last relevant change to the query // (the document falls out) and send us a snapshot that's ahead of // docAv3 (which is already in our cache). - .userListens(visibleQuery, 'resume-token-1000') + .userListens(visibleQuery, { resumeToken: 'resume-token-1000' }) .watchAcks(visibleQuery) .watchSends({ removed: [visibleQuery] }, docAv2) .watchCurrents(visibleQuery, 'resume-token-5000') @@ -362,7 +362,7 @@ describeSpec('Listens:', [], () => { .userUnlistens(visibleQuery) .watchRemoves(visibleQuery) // Listen to allQuery again and make sure we still get docAv3. - .userListens(allQuery, 'resume-token-4000') + .userListens(allQuery, { resumeToken: 'resume-token-4000' }) .expectEvents(allQuery, { added: [docAv3], fromCache: true }) .watchAcksFull(allQuery, 6000) .expectEvents(allQuery, { fromCache: false }) @@ -398,7 +398,7 @@ describeSpec('Listens:', [], () => { // us up to docAV2 since that's the last relevant change to the query // (the document falls out) and send us a snapshot that's ahead of // docAv3 (which is already in our cache). - .userListens(visibleQuery, 'resume-token-1000') + .userListens(visibleQuery, { resumeToken: 'resume-token-1000' }) .watchAcks(visibleQuery) .watchSends({ removed: [visibleQuery] }, docAv2) .watchCurrents(visibleQuery, 'resume-token-5000') @@ -407,7 +407,7 @@ describeSpec('Listens:', [], () => { .userUnlistens(visibleQuery) .watchRemoves(visibleQuery) // Listen to allQuery again and make sure we still get no docs. - .userListens(allQuery, 'resume-token-4000') + .userListens(allQuery, { resumeToken: 'resume-token-4000' }) .watchAcksFull(allQuery, 6000) .expectEvents(allQuery, { fromCache: false }) ); @@ -484,7 +484,7 @@ describeSpec('Listens:', [], () => { .userUnlistens(collQuery) .watchRemoves(collQuery) // Verify that DocA and DocB exists - .userListens(collQuery, 'resume-token-1000') + .userListens(collQuery, { resumeToken: 'resume-token-1000' }) .expectEvents(collQuery, { added: [docA, docB], fromCache: true }) .userUnlistens(collQuery) // Now send a document query that produces no results from the server @@ -498,7 +498,7 @@ describeSpec('Listens:', [], () => { .userUnlistens(docQuery) .watchRemoves(docQuery) // Re-add the initial collection query. Only Doc B exists now - .userListens(collQuery, 'resume-token-1000') + .userListens(collQuery, { resumeToken: 'resume-token-1000' }) .expectEvents(collQuery, { added: [docB], fromCache: true }) ); }); @@ -514,7 +514,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, { added: [docA] }) .userUnlistens(query1) .watchRemoves(query1) - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [docA], fromCache: true }) .watchAcks(query1) .watchSends({ removed: [query1] }, deletedDocA) @@ -536,7 +536,7 @@ describeSpec('Listens:', [], () => { .watchSends({ affects: [query1] }, docB) .watchSnapshots(2000) .watchRemoves(query1) - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [docA], fromCache: true }); }); @@ -560,7 +560,7 @@ describeSpec('Listens:', [], () => { // snapshot and don't synthesize a document delete. .expectEvents(query1, { fromCache: false }) .userUnlistens(query1) - .userListens(query1, 'resume-token-2000') + .userListens(query1, { resumeToken: 'resume-token-2000' }) .expectEvents(query1, { added: [docA], fromCache: true }) ); } @@ -596,7 +596,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, { added: [docA] }) .userUnlistens(query1) .watchRemoves(query1) - .userListens(query1, 'resume-token-2000') + .userListens(query1, { resumeToken: 'resume-token-2000' }) .expectEvents(query1, { added: [docA], fromCache: true }) .watchAcksFull(query1, 3000) .expectEvents(query1, {}); @@ -616,7 +616,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, { added: [docA] }) .userUnlistens(query1) .watchRemoves(query1) - .userListens(query1, 'resume-token-2000') + .userListens(query1, { resumeToken: 'resume-token-2000' }) .expectEvents(query1, { added: [docA], fromCache: true }) .watchAcksFull(query1, 3000) .expectEvents(query1, {}); @@ -639,7 +639,7 @@ describeSpec('Listens:', [], () => { .userUnlistens(query1) .watchRemoves(query1) - .userListens(query1, 'resume-token-2000') + .userListens(query1, { resumeToken: 'resume-token-2000' }) .expectEvents(query1, { added: [docA], fromCache: true }) .watchAcks(query1) .watchCurrents(query1, 'resume-token-3000') @@ -667,7 +667,7 @@ describeSpec('Listens:', [], () => { .watchSnapshots(2000, [], 'resume-token-2000') .restart() - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [docA], fromCache: true }) .watchAcks(query1) .watchCurrents(query1, 'resume-token-3000') @@ -700,7 +700,7 @@ describeSpec('Listens:', [], () => { .watchSnapshots(minutesLater, [], 'resume-token-minutes-later') .restart() - .userListens(query1, 'resume-token-minutes-later') + .userListens(query1, { resumeToken: 'resume-token-minutes-later' }) .expectEvents(query1, { added: [docA], fromCache: true }) .watchAcks(query1) .watchCurrents(query1, 'resume-token-even-later') @@ -862,7 +862,7 @@ describeSpec('Listens:', [], () => { .userListens(query1) .expectEvents(query1, { added: [docA], fromCache: true }) .client(0) - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docB) .client(1) .expectEvents(query1, { added: [docB] }); @@ -1133,7 +1133,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, { fromCache: true }) .client(0) .enableNetwork() - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000) .client(1) .expectEvents(query1, {}); @@ -1184,7 +1184,7 @@ describeSpec('Listens:', [], () => { .client(1) .expectEvents(query1, { fromCache: true }) .client(2) - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .expectPrimaryState(true) .watchAcksFull(query1, 2000) .client(0) @@ -1269,7 +1269,7 @@ describeSpec('Listens:', [], () => { .client(1) .runTimer(TimerId.ClientMetadataRefresh) .expectPrimaryState(true) - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docB) .expectEvents(query1, { added: [docB] }) .client(2) @@ -1299,7 +1299,7 @@ describeSpec('Listens:', [], () => { .client(1) .runTimer(TimerId.ClientMetadataRefresh) .expectPrimaryState(true) - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docB) .client(2) .expectEvents(query1, { added: [docB] }); @@ -1321,7 +1321,7 @@ describeSpec('Listens:', [], () => { .userListens(query1) .expectEvents(query1, { added: [docA] }) .stealPrimaryLease() - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docB) .expectEvents(query1, { added: [docB] }) .client(0) @@ -1359,7 +1359,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, { added: [docA] }) .client(2) .stealPrimaryLease() - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .client(1) .runTimer(TimerId.ClientMetadataRefresh) .expectPrimaryState(false) @@ -1369,7 +1369,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, { added: [docB] }) .client(1) .stealPrimaryLease() - .expectListen(query1, 'resume-token-2000') + .expectListen(query1, { resumeToken: 'resume-token-2000' }) .watchAcksFull(query1, 3000, docC) .client(0) .expectEvents(query1, { added: [docC] }); @@ -1427,7 +1427,7 @@ describeSpec('Listens:', [], () => { .shutdown() .client(2) .expectPrimaryState(true) - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docB) .client(1) .expectEvents(query1, { added: [docB] }); @@ -1448,7 +1448,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, {}) .client(1) .stealPrimaryLease() - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docA) .shutdown() .client(0) @@ -1457,7 +1457,7 @@ describeSpec('Listens:', [], () => { // is already eligible to obtain it again. .runTimer(TimerId.ClientMetadataRefresh) .expectPrimaryState(true) - .expectListen(query1, 'resume-token-2000') + .expectListen(query1, { resumeToken: 'resume-token-2000' }) .expectEvents(query1, { added: [docA] }) ); } diff --git a/packages/firestore/test/unit/specs/orderby_spec.test.ts b/packages/firestore/test/unit/specs/orderby_spec.test.ts index f74e1594ae6..3bb8a4fa0e7 100644 --- a/packages/firestore/test/unit/specs/orderby_spec.test.ts +++ b/packages/firestore/test/unit/specs/orderby_spec.test.ts @@ -66,7 +66,7 @@ describeSpec('OrderBy:', [], () => { .expectEvents(query1, { added: [docB, docA] }) .userUnlistens(query1) .watchRemoves(query1) - .userListens(query1, 'resume-token-1002') + .userListens(query1, { resumeToken: 'resume-token-1002' }) .expectEvents(query1, { added: [docB, docA], fromCache: true }) .watchAcksFull(query1, 1002) .expectEvents(query1, {}); diff --git a/packages/firestore/test/unit/specs/perf_spec.test.ts b/packages/firestore/test/unit/specs/perf_spec.test.ts index 7fafa5e95f1..50346863cd2 100644 --- a/packages/firestore/test/unit/specs/perf_spec.test.ts +++ b/packages/firestore/test/unit/specs/perf_spec.test.ts @@ -234,7 +234,9 @@ describeSpec( .expectEvents(query1, { added: docs }) .userUnlistens(query1) .watchRemoves(query1) - .userListens(query1, 'resume-token-' + currentVersion) + .userListens(query1, { + resumeToken: 'resume-token-' + currentVersion + }) .expectEvents(query1, { added: docs, fromCache: true }) .watchAcksFull(query1, ++currentVersion) .expectEvents(query1, {}) diff --git a/packages/firestore/test/unit/specs/persistence_spec.test.ts b/packages/firestore/test/unit/specs/persistence_spec.test.ts index ac89459ce25..7beaf32d26c 100644 --- a/packages/firestore/test/unit/specs/persistence_spec.test.ts +++ b/packages/firestore/test/unit/specs/persistence_spec.test.ts @@ -79,7 +79,7 @@ describeSpec('Persistence:', [], () => { .watchAcksFull(query1, 1000, doc1) .expectEvents(query1, { added: [doc1] }) .restart() - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [doc1], fromCache: true }); }); @@ -94,7 +94,7 @@ describeSpec('Persistence:', [], () => { .expectEvents(query1, { added: [doc1] }) // Normally this would clear the cached remote documents. .userUnlistens(query1) - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [doc1], fromCache: true }) ); }); diff --git a/packages/firestore/test/unit/specs/recovery_spec.test.ts b/packages/firestore/test/unit/specs/recovery_spec.test.ts index c5da05ff28f..02dfa079d76 100644 --- a/packages/firestore/test/unit/specs/recovery_spec.test.ts +++ b/packages/firestore/test/unit/specs/recovery_spec.test.ts @@ -173,7 +173,7 @@ describeSpec('Persistence Recovery', ['no-ios', 'no-android'], () => { .recoverDatabase() .runTimer(TimerId.AsyncQueueRetry) .expectPrimaryState(true) - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docB) .expectEvents(query1, { added: [docB] }) ); @@ -214,7 +214,7 @@ describeSpec('Persistence Recovery', ['no-ios', 'no-android'], () => { .recoverDatabase() .runTimer(TimerId.AsyncQueueRetry) .expectPrimaryState(true) - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docB) .client(2) .expectEvents(query1, { added: [docB] }) @@ -506,7 +506,7 @@ describeSpec('Persistence Recovery', ['no-ios', 'no-android'], () => { .recoverDatabase() .userUnlistens(query1) // No event since the document was removed - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) ); } ); @@ -612,7 +612,7 @@ describeSpec('Persistence Recovery', ['no-ios', 'no-android'], () => { // Verify that `doc1Query` can be listened to again. Note that the // resume token is slightly outdated since we failed to persist the // target update during the release. - .userListens(doc1Query, 'resume-token-1000') + .userListens(doc1Query, { resumeToken: 'resume-token-1000' }) .expectEvents(doc1Query, { added: [doc1a], fromCache: true @@ -829,7 +829,7 @@ describeSpec('Persistence Recovery', ['no-ios', 'no-android'], () => { .userUnlistens(query1) .watchRemoves(query1) .recoverDatabase() - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [doc1], fromCache: true diff --git a/packages/firestore/test/unit/specs/resume_token_spec.test.ts b/packages/firestore/test/unit/specs/resume_token_spec.test.ts index 537b637fe39..cd0f5b9435a 100644 --- a/packages/firestore/test/unit/specs/resume_token_spec.test.ts +++ b/packages/firestore/test/unit/specs/resume_token_spec.test.ts @@ -51,7 +51,7 @@ describeSpec('Resume tokens:', [], () => { .watchSnapshots(1000) .expectEvents(query1, { added: [doc1] }) .userUnlistens(query1) - .userListens(query1, 'custom-query-resume-token') + .userListens(query1, { resumeToken: 'custom-query-resume-token' }) .expectEvents(query1, { fromCache: true, added: [doc1] }) .watchAcks(query1) .watchSnapshots(1001); diff --git a/packages/firestore/test/unit/specs/spec_builder.ts b/packages/firestore/test/unit/specs/spec_builder.ts index 2d68435bac0..d948586bb5b 100644 --- a/packages/firestore/test/unit/specs/spec_builder.ts +++ b/packages/firestore/test/unit/specs/spec_builder.ts @@ -64,6 +64,7 @@ import { } from './spec_test_runner'; import { UserDataWriter } from '../../../src/api/user_data_writer'; import { firestore } from '../../util/api_helpers'; +import { ResourcePath } from '../../../src/model/path'; const userDataWriter = new UserDataWriter(firestore()); @@ -75,7 +76,8 @@ export interface LimboMap { export interface ActiveTargetSpec { queries: SpecQuery[]; - resumeToken: string; + resumeToken?: string; + readTime?: TestSnapshotVersion; } export interface ActiveTargetMap { @@ -255,7 +257,10 @@ export class SpecBuilder { return this; } - userListens(query: Query, resumeToken?: string): this { + userListens( + query: Query, + resume?: { resumeToken?: string; readTime?: TestSnapshotVersion } + ): this { this.nextStep(); const target = queryToTarget(query); @@ -264,7 +269,7 @@ export class SpecBuilder { if (this.injectFailures) { // Return a `userListens()` step but don't advance the target IDs. this.currentStep = { - userListen: [targetId, SpecBuilder.queryToSpec(query)] + userListen: { targetId, query: SpecBuilder.queryToSpec(query) } }; } else { if (this.queryMapping.has(target)) { @@ -274,9 +279,14 @@ export class SpecBuilder { } this.queryMapping.set(target, targetId); - this.addQueryToActiveTargets(targetId, query, resumeToken); + this.addQueryToActiveTargets( + targetId, + query, + resume?.resumeToken, + resume?.readTime + ); this.currentStep = { - userListen: [targetId, SpecBuilder.queryToSpec(query)], + userListen: { targetId, query: SpecBuilder.queryToSpec(query) }, expectedState: { activeTargets: { ...this.activeTargets } } }; } @@ -363,6 +373,18 @@ export class SpecBuilder { return this; } + loadBundle(bundleContent: string): this { + this.nextStep(); + this.currentStep = { + loadBundle: bundleContent + }; + // Loading a bundle implicitly creates a new target. We advance the `queryIdGenerator` to match. + this.queryIdGenerator.next( + queryToTarget(newQueryForPath(ResourcePath.emptyPath())) + ); + return this; + } + // PORTING NOTE: Only used by web multi-tab tests. becomeHidden(): this { this.nextStep(); @@ -487,13 +509,22 @@ export class SpecBuilder { /** Overrides the currently expected set of active targets. */ expectActiveTargets( - ...targets: Array<{ query: Query; resumeToken?: string }> + ...targets: Array<{ + query: Query; + resumeToken?: string; + readTime?: TestSnapshotVersion; + }> ): this { this.assertStep('Active target expectation requires previous step'); const currentStep = this.currentStep!; this.clientState.activeTargets = {}; - targets.forEach(({ query, resumeToken }) => { - this.addQueryToActiveTargets(this.getTargetId(query), query, resumeToken); + targets.forEach(({ query, resumeToken, readTime }) => { + this.addQueryToActiveTargets( + this.getTargetId(query), + query, + resumeToken, + readTime + ); }); currentStep.expectedState = currentStep.expectedState || {}; currentStep.expectedState.activeTargets = { ...this.activeTargets }; @@ -862,14 +893,22 @@ export class SpecBuilder { } /** Registers a query that is active in another tab. */ - expectListen(query: Query, resumeToken?: string): this { + expectListen( + query: Query, + resume?: { resumeToken?: string; readTime?: TestSnapshotVersion } + ): this { this.assertStep('Expectations require previous step'); const target = queryToTarget(query); const targetId = this.queryIdGenerator.cachedId(target); this.queryMapping.set(target, targetId); - this.addQueryToActiveTargets(targetId, query, resumeToken); + this.addQueryToActiveTargets( + targetId, + query, + resume?.resumeToken, + resume?.readTime + ); const currentStep = this.currentStep!; currentStep.expectedState = currentStep.expectedState || {}; @@ -1038,7 +1077,8 @@ export class SpecBuilder { private addQueryToActiveTargets( targetId: number, query: Query, - resumeToken?: string + resumeToken?: string, + readTime?: TestSnapshotVersion ): void { if (this.activeTargets[targetId]) { const activeQueries = this.activeTargets[targetId].queries; @@ -1050,18 +1090,21 @@ export class SpecBuilder { // `query` is not added yet. this.activeTargets[targetId] = { queries: [SpecBuilder.queryToSpec(query), ...activeQueries], - resumeToken: resumeToken || '' + resumeToken: resumeToken || '', + readTime }; } else { this.activeTargets[targetId] = { queries: activeQueries, - resumeToken: resumeToken || '' + resumeToken: resumeToken || '', + readTime }; } } else { this.activeTargets[targetId] = { queries: [SpecBuilder.queryToSpec(query)], - resumeToken: resumeToken || '' + resumeToken: resumeToken || '', + readTime }; } } diff --git a/packages/firestore/test/unit/specs/spec_test_components.ts b/packages/firestore/test/unit/specs/spec_test_components.ts index 701945ff5a0..b25bfa3a225 100644 --- a/packages/firestore/test/unit/specs/spec_test_components.ts +++ b/packages/firestore/test/unit/specs/spec_test_components.ts @@ -206,7 +206,8 @@ export class MockMemoryOfflineComponentProvider extends MemoryOfflineComponentPr return new MockMemoryPersistence( this.gcEnabled ? MemoryEagerDelegate.factory - : p => new MemoryLruDelegate(p, LruParams.DEFAULT) + : p => new MemoryLruDelegate(p, LruParams.DEFAULT), + newSerializer(cfg.databaseInfo.databaseId) ); } } diff --git a/packages/firestore/test/unit/specs/spec_test_runner.ts b/packages/firestore/test/unit/specs/spec_test_runner.ts index c0bdc21ff49..099489d6373 100644 --- a/packages/firestore/test/unit/specs/spec_test_runner.ts +++ b/packages/firestore/test/unit/specs/spec_test_runner.ts @@ -42,6 +42,7 @@ import { } from '../../../src/core/query'; import { SnapshotVersion } from '../../../src/core/snapshot_version'; import { + syncEngineLoadBundle, activeLimboDocumentResolutions, enqueuedLimboDocumentResolutions, registerPendingWritesCallback, @@ -144,12 +145,17 @@ import { SharedWriteTracker } from './spec_test_components'; import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; +import { BundleReader } from '../../../src/util/bundle_reader'; +import { LoadBundleTask } from '../../../src/api/bundle'; import { encodeBase64 } from '../../../src/platform/base64'; import { FakeDocument, SharedFakeWebStorage, testWindow } from '../../util/test_platform'; +import { toByteStreamReader } from '../../../src/platform/byte_stream_reader'; +import { logWarn } from '../../../src/util/log'; +import { newTextEncoder } from '../../../src/platform/serializer'; const ARBITRARY_SEQUENCE_NUMBER = 2; @@ -364,6 +370,8 @@ abstract class TestRunner { return this.doAddSnapshotsInSyncListener(); } else if ('removeSnapshotsInSyncListener' in step) { return this.doRemoveSnapshotsInSyncListener(); + } else if ('loadBundle' in step) { + return this.doLoadBundle(step.loadBundle!); } else if ('watchAck' in step) { return this.doWatchAck(step.watchAck!); } else if ('watchCurrent' in step) { @@ -417,8 +425,9 @@ abstract class TestRunner { private async doListen(listenSpec: SpecUserListen): Promise { let targetFailed = false; - const querySpec = listenSpec[1]; + const querySpec = listenSpec.query; const query = parseQuery(querySpec); + const aggregator = new EventAggregator(query, e => { if (e.error) { targetFailed = true; @@ -507,6 +516,20 @@ abstract class TestRunner { return Promise.resolve(); } + private async doLoadBundle(bundle: string): Promise { + const reader = new BundleReader( + toByteStreamReader(newTextEncoder().encode(bundle)), + this.serializer + ); + const task = new LoadBundleTask(); + return this.queue.enqueue(async () => { + syncEngineLoadBundle(this.syncEngine, reader, task); + await task.catch(e => { + logWarn(`Loading bundle failed with ${e}`); + }); + }); + } + private doMutations(mutations: Mutation[]): Promise { const documentKeys = mutations.map(val => val.key.path.toString()); const syncEngineCallback = new Deferred(); @@ -1030,18 +1053,24 @@ abstract class TestRunner { // TODO(mcg): populate the purpose of the target once it's possible to // encode that in the spec tests. For now, hard-code that it's a listen // despite the fact that it's not always the right value. - const expectedTarget = toTarget( - this.serializer, - new TargetData( - queryToTarget(parseQuery(expected.queries[0])), - targetId, - TargetPurpose.Listen, - ARBITRARY_SEQUENCE_NUMBER, - SnapshotVersion.min(), - SnapshotVersion.min(), - byteStringFromString(expected.resumeToken) - ) + let targetData = new TargetData( + queryToTarget(parseQuery(expected.queries[0])), + targetId, + TargetPurpose.Listen, + ARBITRARY_SEQUENCE_NUMBER ); + if (expected.resumeToken && expected.resumeToken !== '') { + targetData = targetData.withResumeToken( + byteStringFromString(expected.resumeToken), + SnapshotVersion.min() + ); + } else { + targetData = targetData.withResumeToken( + ByteString.EMPTY_BYTE_STRING, + version(expected.readTime!) + ); + } + const expectedTarget = toTarget(this.serializer, targetData); expect(actualTarget.query).to.deep.equal(expectedTarget.query); expect(actualTarget.targetId).to.equal(expectedTarget.targetId); expect(actualTarget.readTime).to.equal(expectedTarget.readTime); @@ -1312,6 +1341,8 @@ export interface SpecStep { addSnapshotsInSyncListener?: true; /** Unlistens from a SnapshotsInSync event. */ removeSnapshotsInSyncListener?: true; + /** Loads a bundle from a string. */ + loadBundle?: string; /** Ack for a query in the watch stream */ watchAck?: SpecWatchAck; @@ -1397,8 +1428,10 @@ export interface SpecStep { expectedWaitForPendingWritesEvents?: number; } -/** [, ] */ -export type SpecUserListen = [TargetId, string | SpecQuery]; +export interface SpecUserListen { + targetId: TargetId; + query: string | SpecQuery; +} /** [, ] */ export type SpecUserUnlisten = [TargetId, string | SpecQuery]; diff --git a/packages/firestore/test/unit/specs/write_spec.test.ts b/packages/firestore/test/unit/specs/write_spec.test.ts index 3cb8b26c782..7c128e79822 100644 --- a/packages/firestore/test/unit/specs/write_spec.test.ts +++ b/packages/firestore/test/unit/specs/write_spec.test.ts @@ -1068,7 +1068,7 @@ describeSpec('Writes:', [], () => { // Start a new client. DocV1 still has pending writes. .client(1) .stealPrimaryLease() - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .userListens(query2) .expectEvents(query2, { added: [docV1Committed], diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts new file mode 100644 index 00000000000..a0a6151c36d --- /dev/null +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -0,0 +1,263 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { expect, use } from 'chai'; +import * as chaiAsPromised from 'chai-as-promised'; +import { + BundleReader, + SizedBundleElement +} from '../../../src/util/bundle_reader'; +import { toByteStreamReader } from '../../../src/platform/byte_stream_reader'; +import { + doc1String, + doc1MetaString, + doc1Meta, + noDocMetaString, + noDocMeta, + doc2MetaString, + doc2Meta, + limitQueryString, + limitQuery, + limitToLastQuery, + limitToLastQueryString, + meta, + metaString, + doc2String, + doc1, + doc2 +} from './bundle_data'; +import { newTextEncoder } from '../../../src/platform/serializer'; +import { JSON_SERIALIZER } from '../local/persistence_test_helpers'; + +use(chaiAsPromised); + +const encoder = newTextEncoder(); + +/** + * Create a `ReadableStream` from a string. + * + * @param content: Bundle in string. + * @param bytesPerRead: How many bytes to read from the underlying buffer from + * each read through the stream. + */ +export function byteStreamReaderFromString( + content: string, + bytesPerRead: number +): ReadableStreamReader { + const data = encoder.encode(content); + return toByteStreamReader(data, bytesPerRead); +} + +// Testing readableStreamFromString() is working as expected. +describe('byteStreamReaderFromString()', () => { + it('returns a reader stepping readable stream', async () => { + const r = byteStreamReaderFromString('0123456789', 4); + + let result = await r.read(); + expect(result.value).to.deep.equal(encoder.encode('0123')); + expect(result.done).to.be.false; + + result = await r.read(); + expect(result.value).to.deep.equal(encoder.encode('4567')); + expect(result.done).to.be.false; + + result = await r.read(); + expect(result.value).to.deep.equal(encoder.encode('89')); + expect(result.done).to.be.false; + + result = await r.read(); + expect(result.value).to.be.undefined; + expect(result.done).to.be.true; + }); +}); + +describe('Bundle ', () => { + genericBundleReadingTests(1); + genericBundleReadingTests(4); + genericBundleReadingTests(64); + genericBundleReadingTests(1024); +}); + +function genericBundleReadingTests(bytesPerRead: number): void { + function bundleFromString(s: string): BundleReader { + return new BundleReader( + byteStreamReaderFromString(s, bytesPerRead), + JSON_SERIALIZER + ); + } + + async function getAllElements( + bundle: BundleReader + ): Promise { + const result: SizedBundleElement[] = []; + while (true) { + const sizedElement = await bundle.nextElement(); + if (sizedElement === null) { + break; + } + if (!sizedElement.isBundleMetadata()) { + result.push(sizedElement); + } + } + + return Promise.resolve(result); + } + + function verifySizedElement( + element: SizedBundleElement, + payload: unknown, + payloadString: string + ): void { + expect(element.payload).to.deep.equal(payload); + expect(element.byteLength).to.equal( + encoder.encode(payloadString).byteLength + ); + } + + async function generateBundleAndParse( + bundleString: string, + bytesPerRead: number, + validMeta = false + ): Promise { + const bundle = bundleFromString(bundleString); + + if (!validMeta) { + await expect(await bundle.getMetadata()).should.be.rejected; + } else { + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + } + + await getAllElements(bundle); + } + + it('reads with query and doc with bytesPerRead ' + bytesPerRead, async () => { + const bundle = bundleFromString( + metaString + + limitQueryString + + limitToLastQueryString + + doc1MetaString + + doc1String + ); + + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + + const actual = await getAllElements(bundle); + expect(actual.length).to.equal(4); + verifySizedElement(actual[0], limitQuery, limitQueryString); + verifySizedElement(actual[1], limitToLastQuery, limitToLastQueryString); + verifySizedElement(actual[2], doc1Meta, doc1MetaString); + verifySizedElement(actual[3], doc1, doc1String); + }); + + it( + 'reads with unexpected orders with bytesPerRead ' + bytesPerRead, + async () => { + const bundle = bundleFromString( + metaString + + doc1MetaString + + doc1String + + limitQueryString + + doc2MetaString + + doc2String + ); + + const actual = await getAllElements(bundle); + expect(actual.length).to.equal(5); + verifySizedElement(actual[0], doc1Meta, doc1MetaString); + verifySizedElement(actual[1], doc1, doc1String); + verifySizedElement(actual[2], limitQuery, limitQueryString); + verifySizedElement(actual[3], doc2Meta, doc2MetaString); + verifySizedElement(actual[4], doc2, doc2String); + + // Reading metadata after other elements should also work. + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + } + ); + + it( + 'reads without named query with bytesPerRead ' + bytesPerRead, + async () => { + const bundle = bundleFromString(metaString + doc1MetaString + doc1String); + + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + + const actual = await getAllElements(bundle); + expect(actual.length).to.equal(2); + verifySizedElement(actual[0], doc1Meta, doc1MetaString); + verifySizedElement(actual[1], doc1, doc1String); + } + ); + + it('reads with deleted doc with bytesPerRead ' + bytesPerRead, async () => { + const bundle = bundleFromString( + metaString + noDocMetaString + doc1MetaString + doc1String + ); + + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + + const actual = await getAllElements(bundle); + expect(actual.length).to.equal(3); + verifySizedElement(actual[0], noDocMeta, noDocMetaString); + verifySizedElement(actual[1], doc1Meta, doc1MetaString); + verifySizedElement(actual[2], doc1, doc1String); + }); + + it( + 'reads without documents or query with bytesPerRead ' + bytesPerRead, + async () => { + const bundle = bundleFromString(metaString); + + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + + const actual = await getAllElements(bundle); + expect(actual.length).to.equal(0); + } + ); + + it( + 'throws with ill-formatted bundle with bytesPerRead ' + bytesPerRead, + async () => { + await expect( + generateBundleAndParse('metadata: "no length prefix"', bytesPerRead) + ).to.be.rejectedWith( + 'Reached the end of bundle when a length string is expected.' + ); + + await expect( + generateBundleAndParse('{metadata: "no length prefix"}', bytesPerRead) + ).to.be.rejectedWith('Unexpected end of JSON input'); + + await expect( + generateBundleAndParse( + metaString + 'invalid-string', + bytesPerRead, + true + ) + ).to.be.rejectedWith( + 'Reached the end of bundle when a length string is expected.' + ); + + await expect( + generateBundleAndParse('1' + metaString, bytesPerRead) + ).to.be.rejectedWith('Reached the end of bundle when more is expected.'); + + // First element is not BundleMetadata. + await expect( + generateBundleAndParse(doc1MetaString + doc1String, bytesPerRead) + ).to.be.rejectedWith('The first element of the bundle is not a metadata'); + } + ); +} diff --git a/packages/firestore/test/unit/util/bundle_data.ts b/packages/firestore/test/unit/util/bundle_data.ts new file mode 100644 index 00000000000..386256e0e3b --- /dev/null +++ b/packages/firestore/test/unit/util/bundle_data.ts @@ -0,0 +1,247 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + BundleElement, + LimitType as BundleLimitType +} from '../../../src/protos/firestore_bundle_proto'; +import { DatabaseId } from '../../../src/core/database_info'; +import * as api from '../../../src/protos/firestore_proto_api'; +import { Value } from '../../../src/protos/firestore_proto_api'; +import { + JsonProtoSerializer, + toName, + toQueryTarget +} from '../../../src/remote/serializer'; +import { DocumentKey } from '../../../src/model/document_key'; +import { + newSerializer, + newTextEncoder +} from '../../../src/platform/serializer'; +import { + LimitType, + Query, + queryToTarget, + queryWithLimit +} from '../../../src/core/query'; + +export const encoder = newTextEncoder(); + +function lengthPrefixedString(o: {}): string { + const str = JSON.stringify(o); + const l = encoder.encode(str).byteLength; + return `${l}${str}`; +} + +export class TestBundleBuilder { + readonly elements: BundleElement[] = []; + private serializer: JsonProtoSerializer; + constructor(private databaseId: DatabaseId) { + this.serializer = newSerializer(databaseId); + } + + addDocumentMetadata( + docKey: DocumentKey, + readTime: api.Timestamp, + exists: boolean + ): TestBundleBuilder { + this.elements.push({ + documentMetadata: { + name: toName(this.serializer, docKey), + readTime, + exists + } + }); + return this; + } + addDocument( + docKey: DocumentKey, + createTime: api.Timestamp, + updateTime: api.Timestamp, + fields: api.ApiClientObjectMap + ): TestBundleBuilder { + this.elements.push({ + document: { + name: toName(this.serializer, docKey), + createTime, + updateTime, + fields + } + }); + return this; + } + + addNamedQuery( + name: string, + readTime: api.Timestamp, + query: Query + ): TestBundleBuilder { + let bundledLimitType: BundleLimitType | undefined = !!query.limit + ? 'FIRST' + : undefined; + if (query.limitType === LimitType.Last) { + query = queryWithLimit(query, query.limit!, LimitType.First); + bundledLimitType = 'LAST'; + } + const queryTarget = toQueryTarget(this.serializer, queryToTarget(query)); + this.elements.push({ + namedQuery: { + name, + readTime, + bundledQuery: { + parent: queryTarget.parent, + structuredQuery: queryTarget.structuredQuery, + limitType: bundledLimitType + } + } + }); + return this; + } + + getMetadataElement( + id: string, + createTime: api.Timestamp, + version = 1 + ): BundleElement { + let totalDocuments = 0; + let totalBytes = 0; + for (const element of this.elements) { + if (element.documentMetadata && !element.documentMetadata.exists) { + totalDocuments += 1; + } + if (element.document) { + totalDocuments += 1; + } + totalBytes += encoder.encode(lengthPrefixedString(element)).byteLength; + } + + return { + metadata: { + id, + createTime, + version, + totalDocuments, + totalBytes + } + }; + } + + build(id: string, createTime: api.Timestamp, version = 1): string { + let result = ''; + for (const element of this.elements) { + result += lengthPrefixedString(element); + } + return ( + lengthPrefixedString(this.getMetadataElement(id, createTime, version)) + + result + ); + } +} + +// TODO(wuandy): Ideally, these should use `TestBundleBuilder` above. +export const meta: BundleElement = { + metadata: { + id: 'test-bundle', + createTime: { seconds: 1577836805, nanos: 6 }, + version: 1, + totalDocuments: 1, + totalBytes: 416 + } +}; +export const metaString = lengthPrefixedString(meta); + +export const doc1Meta: BundleElement = { + documentMetadata: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc1', + readTime: { seconds: 5, nanos: 6 }, + exists: true + } +}; +export const doc1MetaString = lengthPrefixedString(doc1Meta); +export const doc1: BundleElement = { + document: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc1', + createTime: { seconds: 1, nanos: 2000000 }, + updateTime: { seconds: 3, nanos: 4000 }, + fields: { foo: { stringValue: 'value' }, bar: { integerValue: -42 } } + } +}; +export const doc1String = lengthPrefixedString(doc1); + +export const doc2Meta: BundleElement = { + documentMetadata: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc2', + readTime: { seconds: 5, nanos: 6 }, + exists: true + } +}; +export const doc2MetaString = lengthPrefixedString(doc2Meta); +export const doc2: BundleElement = { + document: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc2', + createTime: { seconds: 1, nanos: 2000000 }, + updateTime: { seconds: 3, nanos: 4000 }, + fields: { foo: { stringValue: 'value1' }, bar: { integerValue: 42 } } + } +}; +export const doc2String = lengthPrefixedString(doc2); + +export const noDocMeta: BundleElement = { + documentMetadata: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/nodoc', + readTime: { seconds: 5, nanos: 6 }, + exists: false + } +}; +export const noDocMetaString = lengthPrefixedString(noDocMeta); + +export const limitQuery: BundleElement = { + namedQuery: { + name: 'limitQuery', + bundledQuery: { + parent: 'projects/fireeats-97d5e/databases/(default)/documents', + structuredQuery: { + from: [{ collectionId: 'node_3.7.5_7Li7XoCjutvNxwD0tpo9' }], + orderBy: [{ field: { fieldPath: 'sort' }, direction: 'DESCENDING' }], + limit: { 'value': 1 } + }, + limitType: 'FIRST' + }, + readTime: { 'seconds': 1590011379, 'nanos': 191164000 } + } +}; +export const limitQueryString = lengthPrefixedString(limitQuery); +export const limitToLastQuery: BundleElement = { + namedQuery: { + name: 'limitToLastQuery', + bundledQuery: { + parent: 'projects/fireeats-97d5e/databases/(default)/documents', + structuredQuery: { + from: [{ collectionId: 'node_3.7.5_7Li7XoCjutvNxwD0tpo9' }], + orderBy: [{ field: { fieldPath: 'sort' }, direction: 'ASCENDING' }], + limit: { 'value': 1 } + }, + limitType: 'LAST' + }, + readTime: { 'seconds': 1590011379, 'nanos': 543063000 } + } +}; +export const limitToLastQueryString = lengthPrefixedString(limitToLastQuery); diff --git a/packages/firestore/test/util/helpers.ts b/packages/firestore/test/util/helpers.ts index 4c11d378616..a09c92ca1a9 100644 --- a/packages/firestore/test/util/helpers.ts +++ b/packages/firestore/test/util/helpers.ts @@ -95,12 +95,24 @@ import { SortedSet } from '../../src/util/sorted_set'; import { FIRESTORE } from './api_helpers'; import { ByteString } from '../../src/util/byte_string'; import { decodeBase64, encodeBase64 } from '../../src/platform/base64'; -import { JsonProtoSerializer } from '../../src/remote/serializer'; +import { + JsonProtoSerializer, + toDocument, + toName, + toQueryTarget, + toTimestamp, + toVersion +} from '../../src/remote/serializer'; import { Timestamp } from '../../src/api/timestamp'; import { DocumentReference } from '../../src/api/database'; import { DeleteFieldValueImpl } from '../../src/api/field_value'; import { Code, FirestoreError } from '../../src/util/error'; -import { TEST_DATABASE_ID } from '../unit/local/persistence_test_helpers'; +import { + JSON_SERIALIZER, + TEST_DATABASE_ID +} from '../unit/local/persistence_test_helpers'; +import { BundledDocuments } from '../../src/core/bundle'; +import * as bundleProto from '../../src/protos/firestore_bundle_proto'; /* eslint-disable no-restricted-globals */ @@ -417,6 +429,76 @@ export function docUpdateRemoteEvent( return aggregator.createRemoteEvent(doc.version); } +export class TestBundledDocuments { + constructor(public documents: BundledDocuments, public bundleName: string) {} +} + +export function bundledDocuments( + documents: MaybeDocument[], + queryNames?: string[][], + bundleName?: string +): TestBundledDocuments { + const result = documents.map((d, index) => { + return { + metadata: { + name: toName(JSON_SERIALIZER, d.key), + readTime: toVersion(JSON_SERIALIZER, d.version), + exists: d instanceof Document, + queries: queryNames ? queryNames[index] : undefined + }, + document: + d instanceof Document ? toDocument(JSON_SERIALIZER, d) : undefined + }; + }); + + return new TestBundledDocuments(result, bundleName || ''); +} + +export class TestNamedQuery { + constructor( + public namedQuery: bundleProto.NamedQuery, + public matchingDocuments: DocumentKeySet + ) {} +} + +export function namedQuery( + name: string, + query: Query, + limitType: bundleProto.LimitType, + readTime: SnapshotVersion, + matchingDocuments: DocumentKeySet = documentKeySet() +): TestNamedQuery { + return { + namedQuery: { + name, + readTime: toTimestamp(JSON_SERIALIZER, readTime.toTimestamp()), + bundledQuery: { + parent: toQueryTarget(JSON_SERIALIZER, queryToTarget(query)).parent, + limitType, + structuredQuery: toQueryTarget(JSON_SERIALIZER, queryToTarget(query)) + .structuredQuery + } + }, + matchingDocuments + }; +} + +export function bundleMetadata( + id: string, + createTime: TestSnapshotVersion, + version = 1, + totalDocuments = 1, + totalBytes = 1000 +): bundleProto.BundleMetadata { + return { + id, + createTime: { seconds: createTime, nanos: 0 }, + version, + totalDocuments, + totalBytes + }; +} + export function updateMapping( snapshotVersion: SnapshotVersion, added: Array,