From 9602712941d2a802d5b52f808bc9f8225c2a3681 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 15 May 2020 15:26:07 -0400 Subject: [PATCH 01/43] Renaming interfaces without leading I --- packages/firestore/src/local/indexeddb_schema.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/firestore/src/local/indexeddb_schema.ts b/packages/firestore/src/local/indexeddb_schema.ts index 2026b373adb..645aa7ec226 100644 --- a/packages/firestore/src/local/indexeddb_schema.ts +++ b/packages/firestore/src/local/indexeddb_schema.ts @@ -19,6 +19,7 @@ import { BatchId, ListenSequenceNumber, TargetId } from '../core/types'; import { ResourcePath } from '../model/path'; import { BundledQuery } from '../protos/firestore_bundle_proto'; import * as api from '../protos/firestore_proto_api'; +import { BundledQuery } from '../protos/firestore_bundle_proto'; import { hardAssert, debugAssert } from '../util/assert'; import { SnapshotVersion } from '../core/snapshot_version'; From c5e783ec310164486819a855ead96b4a2e95297c Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 21 May 2020 12:08:35 -0400 Subject: [PATCH 02/43] Initial commit of bundle reading - for web only. --- packages/firestore/src/util/bundle.ts | 215 +++++++++++ .../firestore/test/unit/util/bundle.test.ts | 353 ++++++++++++++++++ 2 files changed, 568 insertions(+) create mode 100644 packages/firestore/src/util/bundle.ts create mode 100644 packages/firestore/test/unit/util/bundle.test.ts diff --git a/packages/firestore/src/util/bundle.ts b/packages/firestore/src/util/bundle.ts new file mode 100644 index 00000000000..d11613361b0 --- /dev/null +++ b/packages/firestore/src/util/bundle.ts @@ -0,0 +1,215 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as api from '../protos/firestore_proto_api'; +import { + BundledDocumentMetadata, + BundledQuery, + BundleElement, + BundleMetadata +} from '../protos/firestore_bundle_proto'; + +/** + * A complete element in the bundle stream, together with the byte length it + * occupies in the stream. + */ +export class SizedBundleElement { + constructor( + public payload: BundledQuery | api.Document | BundledDocumentMetadata, + public byteLength: number + ) {} +} + +/** + * Create a `ReadableStream` from a underlying buffer. + * + * @param data: Underlying buffer. + * @param bytesPerRead: How many bytes to read from the underlying buffer from each read through the stream. + */ +export function toReadableStream( + data: Uint8Array | ArrayBuffer, + bytesPerRead = 10240 +): ReadableStream { + let readFrom = 0; + return new ReadableStream({ + start(controller) {}, + async pull(controller): Promise { + controller.enqueue(data.slice(readFrom, readFrom + bytesPerRead)); + readFrom += bytesPerRead; + if (readFrom >= data.byteLength) { + controller.close(); + } + } + }); +} + +/** + * A class representing a bundle. + * + * Takes a bundle stream or buffer, and presents abstractions to read bundled + * elements out of the underlying content. + */ +export class Bundle { + // Cached bundle metadata. + private metadata?: BundleMetadata | null; + // The reader instance of the given ReadableStream. + private reader: ReadableStreamDefaultReader; + // Internal buffer to hold bundle content, accumulating incomplete element content. + private buffer: Uint8Array = new Uint8Array(); + private textDecoder = new TextDecoder('utf-8'); + + constructor( + private bundleStream: + | ReadableStream + | Uint8Array + | ArrayBuffer + ) { + if ( + bundleStream instanceof Uint8Array || + bundleStream instanceof ArrayBuffer + ) { + this.bundleStream = toReadableStream(bundleStream); + } + this.reader = (this.bundleStream as ReadableStream).getReader(); + } + + /** + * Returns the metadata of the bundle. + */ + async getMetadata(): Promise { + if (!this.metadata) { + const result = await this.nextElement(); + if (result === null || result instanceof SizedBundleElement) { + throw new Error(`The first element is not metadata, it is ${result}`); + } + this.metadata = (result as BundleElement).metadata; + } + + return this.metadata!; + } + + /** + * Asynchronously iterate through all bundle elements (except bundle metadata). + */ + async *elements(): AsyncIterableIterator { + let element = await this.nextElement(); + while (element !== null) { + if (element instanceof SizedBundleElement) { + yield element; + } else { + this.metadata = element.metadata; + } + element = await this.nextElement(); + } + } + + // Reads from the head of internal buffer, and pulling more data from underlying stream if a complete element + // cannot be found, until an element(including the prefixed length and the JSON string) is found. + // + // Once a complete element is read, it is dropped from internal buffer. + // + // Returns either the bundled element, or null if we have reached the end of the stream. + private async nextElement(): Promise< + BundleElement | SizedBundleElement | null + > { + const lengthBuffer = await this.readLength(); + if (lengthBuffer === null) { + return null; + } + + const lengthString = this.textDecoder.decode(lengthBuffer); + const length = parseInt(lengthString, 10); + if (isNaN(length)) { + throw new Error(`length string (${lengthString}) is not valid number`); + } + + const jsonString = await this.readJsonString(lengthBuffer.length, length); + // Update the internal buffer to drop the read length and json string. + this.buffer = this.buffer.slice(lengthBuffer.length + length); + + if (!this.metadata) { + const element = JSON.parse(jsonString) as BundleElement; + return element; + } else { + return new SizedBundleElement( + JSON.parse(jsonString), + lengthBuffer.length + length + ); + } + } + + // First index of '{' from the underlying buffer. + private indexOfOpenBracket(): number { + return this.buffer.findIndex(v => v === 123); + } + + // Reads from the beginning of the inernal buffer, until the first '{', and return + // the content. + // If reached end of the stream, returns a null. + private async readLength(): Promise { + let position = this.indexOfOpenBracket(); + while (position < 0) { + const done = await this.pullMoreDataToBuffer(); + if (done) { + if (this.buffer.length === 0) { + return null; + } + position = this.indexOfOpenBracket(); + // Underlying stream is closed, and we still cannot find a '{'. + if (position < 0) { + throw new Error( + 'Reach to the end of bundle when a length string is expected.' + ); + } + } else { + position = this.indexOfOpenBracket(); + } + } + + return this.buffer.slice(0, position); + } + + // Reads from a specified position from the internal buffer, for a specified + // number of bytes, pulling more data from the underlying stream if needed. + // + // Returns a string decoded from the read bytes. + private async readJsonString(start: number, length: number): Promise { + while (this.buffer.length < start + length) { + const done = await this.pullMoreDataToBuffer(); + if (done) { + throw new Error('Reach to the end of bundle when more is expected.'); + } + } + + return this.textDecoder.decode(this.buffer.slice(start, start + length)); + } + + // Pulls more data from underlying stream to internal buffer. + // Returns a boolean indicating whether the stream is finished. + private async pullMoreDataToBuffer(): Promise { + const result = await this.reader.read(); + if (!result.done) { + const newBuffer = new Uint8Array( + this.buffer.length + result.value.length + ); + newBuffer.set(this.buffer); + newBuffer.set(result.value, this.buffer.length); + this.buffer = newBuffer; + } + return result.done; + } +} diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts new file mode 100644 index 00000000000..20b2c464ff9 --- /dev/null +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -0,0 +1,353 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { expect } from 'chai'; +import { Bundle, toReadableStream } from '../../../src/util/bundle'; + +const encoder = new TextEncoder(); + +function readableStreamFromString( + content: string, + bytesPerRead: number +): ReadableStream { + return toReadableStream(encoder.encode(content), bytesPerRead); +} + +function lengthPrefixedString(o: {}): string { + const str = JSON.stringify(o); + const l = encoder.encode(str).byteLength; + return `${l}${str}`; +} + +describe('readableStreamFromString()', () => { + it('returns stepping readable stream', async () => { + const encoder = new TextEncoder(); + const s = readableStreamFromString('0123456789', 4); + const r = s.getReader(); + + let result = await r.read(); + expect(result.value).to.deep.equal(encoder.encode('0123')); + expect(result.done).to.be.false; + + result = await r.read(); + expect(result.value).to.deep.equal(encoder.encode('4567')); + expect(result.done).to.be.false; + + result = await r.read(); + expect(result.value).to.deep.equal(encoder.encode('89')); + expect(result.done).to.be.false; + + result = await r.read(); + expect(result.value).to.be.undefined; + expect(result.done).to.be.true; + }); +}); + +describe.only('Bundle ', () => { + genericBundleReadingTests(1); + genericBundleReadingTests(4); + genericBundleReadingTests(64); + genericBundleReadingTests(1024); +}); + +function genericBundleReadingTests(bytesPerRead: number): void { + // Setting up test data. + const meta = { + metadata: { + id: 'test-bundle', + createTime: { seconds: 1577836805, nanos: 6 }, + version: 1, + totalDocuments: 1, + totalBytes: 416 + } + }; + const metaString = lengthPrefixedString(meta); + + const doc1Meta = { + documentMetadata: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc1', + readTime: { seconds: 5, nanos: 6 }, + exists: true + } + }; + const doc1MetaString = lengthPrefixedString(doc1Meta); + const doc1 = { + document: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc1', + createTime: { _seconds: 1, _nanoseconds: 2000000 }, + updateTime: { _seconds: 3, _nanoseconds: 4000 }, + fields: { foo: { stringValue: 'value' }, bar: { integerValue: -42 } } + } + }; + const doc1String = lengthPrefixedString(doc1); + + const doc2Meta = { + documentMetadata: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc2', + readTime: { seconds: 5, nanos: 6 }, + exists: true + } + }; + const doc2MetaString = lengthPrefixedString(doc2Meta); + const doc2 = { + document: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc2', + createTime: { _seconds: 1, _nanoseconds: 2000000 }, + updateTime: { _seconds: 3, _nanoseconds: 4000 }, + fields: { foo: { stringValue: 'value1' }, bar: { integerValue: 42 } } + } + }; + const doc2String = lengthPrefixedString(doc2); + + const noDocMeta = { + documentMetadata: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/nodoc', + readTime: { seconds: 5, nanos: 6 }, + exists: false + } + }; + const noDocMetaString = lengthPrefixedString(noDocMeta); + + const limitQuery = { + namedQuery: { + name: 'limitQuery', + bundledQuery: { + parent: 'projects/fireeats-97d5e/databases/(default)/documents', + structuredQuery: { + from: [{ collectionId: 'node_3.7.5_7Li7XoCjutvNxwD0tpo9' }], + orderBy: [{ field: { fieldPath: 'sort' }, direction: 'DESCENDING' }], + limit: { 'value': 1 } + }, + limitType: 'FIRST' + }, + readTime: { 'seconds': 1590011379, 'nanos': 191164000 } + } + }; + const limitQueryString = lengthPrefixedString(limitQuery); + const limitToLastQuery = { + namedQuery: { + name: 'limitToLastQuery', + bundledQuery: { + parent: 'projects/fireeats-97d5e/databases/(default)/documents', + structuredQuery: { + from: [{ collectionId: 'node_3.7.5_7Li7XoCjutvNxwD0tpo9' }], + orderBy: [{ field: { fieldPath: 'sort' }, direction: 'ASCENDING' }], + limit: { 'value': 1 } + }, + limitType: 'LAST' + }, + readTime: { 'seconds': 1590011379, 'nanos': 543063000 } + } + }; + const limitToLastQueryString = lengthPrefixedString(limitToLastQuery); + + async function expectErrorFromBundle( + bundleString: string, + bytesPerRead: number, + validMeta = false + ): Promise { + const bundleStream = readableStreamFromString(bundleString, bytesPerRead); + const bundle = new Bundle(bundleStream); + + if (!validMeta) { + await expect(await bundle.getMetadata()).should.be.rejected; + } else { + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + } + + const actual = []; + for await (const sizedElement of bundle.elements()) { + actual.push(sizedElement); + } + } + + it('reads with query and doc with bytesPerRead ' + bytesPerRead, async () => { + const bundleStream = readableStreamFromString( + metaString + + limitQueryString + + limitToLastQueryString + + doc1MetaString + + doc1String, + bytesPerRead + ); + const bundle = new Bundle(bundleStream); + + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + + const actual = []; + for await (const sizedElement of bundle.elements()) { + actual.push(sizedElement); + } + expect(actual.length).to.equal(4); + expect(actual[0]).to.deep.equal({ + payload: limitQuery, + byteLength: encoder.encode(limitQueryString).byteLength + }); + expect(actual[1]).to.deep.equal({ + payload: limitToLastQuery, + byteLength: encoder.encode(limitToLastQueryString).byteLength + }); + expect(actual[2]).to.deep.equal({ + payload: doc1Meta, + byteLength: encoder.encode(doc1MetaString).byteLength + }); + expect(actual[3]).to.deep.equal({ + payload: doc1, + byteLength: encoder.encode(doc1String).byteLength + }); + }); + + it( + 'reads with unexpected orders with bytesPerRead ' + bytesPerRead, + async () => { + const bundleStream = readableStreamFromString( + metaString + + doc1MetaString + + doc1String + + limitQueryString + + doc2MetaString + + doc2String, + bytesPerRead + ); + const bundle = new Bundle(bundleStream); + + const actual = []; + for await (const sizedElement of bundle.elements()) { + actual.push(sizedElement); + } + expect(actual.length).to.equal(5); + expect(actual[0]).to.deep.equal({ + payload: doc1Meta, + byteLength: encoder.encode(doc1MetaString).byteLength + }); + expect(actual[1]).to.deep.equal({ + payload: doc1, + byteLength: encoder.encode(doc1String).byteLength + }); + expect(actual[2]).to.deep.equal({ + payload: limitQuery, + byteLength: encoder.encode(limitQueryString).byteLength + }); + expect(actual[3]).to.deep.equal({ + payload: doc2Meta, + byteLength: encoder.encode(doc2MetaString).byteLength + }); + expect(actual[4]).to.deep.equal({ + payload: doc2, + byteLength: encoder.encode(doc2String).byteLength + }); + + // Reading metadata after other elements should also work. + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + } + ); + + it( + 'reads without named query with bytesPerRead ' + bytesPerRead, + async () => { + const bundleStream = readableStreamFromString( + metaString + doc1MetaString + doc1String, + bytesPerRead + ); + const bundle = new Bundle(bundleStream); + + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + + const actual = []; + for await (const sizedElement of bundle.elements()) { + actual.push(sizedElement); + } + expect(actual.length).to.equal(2); + expect(actual[0]).to.deep.equal({ + payload: doc1Meta, + byteLength: encoder.encode(doc1MetaString).byteLength + }); + expect(actual[1]).to.deep.equal({ + payload: doc1, + byteLength: encoder.encode(doc1String).byteLength + }); + } + ); + + it('reads with deleted doc with bytesPerRead ' + bytesPerRead, async () => { + const bundleStream = readableStreamFromString( + metaString + noDocMetaString + doc1MetaString + doc1String, + bytesPerRead + ); + const bundle = new Bundle(bundleStream); + + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + + const actual = []; + for await (const sizedElement of bundle.elements()) { + actual.push(sizedElement); + } + expect(actual.length).to.equal(3); + expect(actual[0]).to.deep.equal({ + payload: noDocMeta, + byteLength: encoder.encode(noDocMetaString).byteLength + }); + expect(actual[1]).to.deep.equal({ + payload: doc1Meta, + byteLength: encoder.encode(doc1MetaString).byteLength + }); + expect(actual[2]).to.deep.equal({ + payload: doc1, + byteLength: encoder.encode(doc1String).byteLength + }); + }); + + it( + 'reads without documents or query with bytesPerRead ' + bytesPerRead, + async () => { + const bundleStream = readableStreamFromString(metaString, bytesPerRead); + const bundle = new Bundle(bundleStream); + + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + + const actual = []; + for await (const sizedElement of bundle.elements()) { + actual.push(sizedElement); + } + expect(actual.length).to.equal(0); + } + ); + + it( + 'throws with ill-formatted bundle with bytesPerRead ' + bytesPerRead, + async () => { + await expect( + expectErrorFromBundle('metadata: "no length prefix"', bytesPerRead) + ).to.be.rejected; + + await expect( + expectErrorFromBundle('{metadata: "no length prefix"}', bytesPerRead) + ).to.be.rejected; + + await expect( + expectErrorFromBundle(metaString + 'invalid-string', bytesPerRead, true) + ).to.be.rejected; + + await expect(expectErrorFromBundle('1' + metaString, bytesPerRead)).to.be + .rejected; + } + ); +} From 5e7fb89aad81aa52661bb1c4e419da8c32aef6d5 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 21 May 2020 13:25:12 -0400 Subject: [PATCH 03/43] Tests only run when it is not Node. --- packages/firestore/test/unit/util/bundle.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index 20b2c464ff9..6083bd81a7d 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -16,6 +16,7 @@ */ import { expect } from 'chai'; import { Bundle, toReadableStream } from '../../../src/util/bundle'; +import { isNode } from '../../util/test_platform'; const encoder = new TextEncoder(); @@ -32,7 +33,8 @@ function lengthPrefixedString(o: {}): string { return `${l}${str}`; } -describe('readableStreamFromString()', () => { +// eslint-disable-next-line no-restricted-properties +(isNode() ? describe.skip : describe)('readableStreamFromString()', () => { it('returns stepping readable stream', async () => { const encoder = new TextEncoder(); const s = readableStreamFromString('0123456789', 4); @@ -56,7 +58,8 @@ describe('readableStreamFromString()', () => { }); }); -describe.only('Bundle ', () => { +// eslint-disable-next-line no-restricted-properties +(isNode() ? describe.skip : describe)('Bundle ', () => { genericBundleReadingTests(1); genericBundleReadingTests(4); genericBundleReadingTests(64); From 1ee161546c90869ac4105ced02656831b1e6e63a Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 21 May 2020 13:51:41 -0400 Subject: [PATCH 04/43] Fix redundant imports --- packages/firestore/src/local/indexeddb_schema.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/firestore/src/local/indexeddb_schema.ts b/packages/firestore/src/local/indexeddb_schema.ts index 645aa7ec226..2026b373adb 100644 --- a/packages/firestore/src/local/indexeddb_schema.ts +++ b/packages/firestore/src/local/indexeddb_schema.ts @@ -19,7 +19,6 @@ import { BatchId, ListenSequenceNumber, TargetId } from '../core/types'; import { ResourcePath } from '../model/path'; import { BundledQuery } from '../protos/firestore_bundle_proto'; import * as api from '../protos/firestore_proto_api'; -import { BundledQuery } from '../protos/firestore_bundle_proto'; import { hardAssert, debugAssert } from '../util/assert'; import { SnapshotVersion } from '../core/snapshot_version'; From 18f0be1d843adbc4f2ebe921111307fa344f2f03 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 21 May 2020 14:35:54 -0400 Subject: [PATCH 05/43] Fix missing textencoder --- packages/firestore/test/unit/util/bundle.test.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index 6083bd81a7d..39c705dc647 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -18,18 +18,16 @@ import { expect } from 'chai'; import { Bundle, toReadableStream } from '../../../src/util/bundle'; import { isNode } from '../../util/test_platform'; -const encoder = new TextEncoder(); - function readableStreamFromString( content: string, bytesPerRead: number ): ReadableStream { - return toReadableStream(encoder.encode(content), bytesPerRead); + return toReadableStream(new TextEncoder().encode(content), bytesPerRead); } function lengthPrefixedString(o: {}): string { const str = JSON.stringify(o); - const l = encoder.encode(str).byteLength; + const l = new TextEncoder().encode(str).byteLength; return `${l}${str}`; } From aa455bff1db26c14b37914f202ddf59cb55e4965 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 29 May 2020 13:59:20 -0400 Subject: [PATCH 06/43] Remove generator. --- .../src/util/{bundle.ts => bundle_reader.ts} | 114 ++++++------ .../firestore/test/unit/util/bundle.test.ts | 162 ++++++++---------- 2 files changed, 134 insertions(+), 142 deletions(-) rename packages/firestore/src/util/{bundle.ts => bundle_reader.ts} (66%) diff --git a/packages/firestore/src/util/bundle.ts b/packages/firestore/src/util/bundle_reader.ts similarity index 66% rename from packages/firestore/src/util/bundle.ts rename to packages/firestore/src/util/bundle_reader.ts index d11613361b0..8bfb19f8e5d 100644 --- a/packages/firestore/src/util/bundle.ts +++ b/packages/firestore/src/util/bundle_reader.ts @@ -15,10 +15,7 @@ * limitations under the License. */ -import * as api from '../protos/firestore_proto_api'; import { - BundledDocumentMetadata, - BundledQuery, BundleElement, BundleMetadata } from '../protos/firestore_bundle_proto'; @@ -29,9 +26,14 @@ import { */ export class SizedBundleElement { constructor( - public payload: BundledQuery | api.Document | BundledDocumentMetadata, - public byteLength: number + public readonly payload: BundleElement, + // How many bytes this element takes to store in the bundle. + public readonly byteLength: number ) {} + + isBundleMetadata(): boolean { + return 'metadata' in this.payload; + } } /** @@ -63,7 +65,7 @@ export function toReadableStream( * Takes a bundle stream or buffer, and presents abstractions to read bundled * elements out of the underlying content. */ -export class Bundle { +export class BundleReader { // Cached bundle metadata. private metadata?: BundleMetadata | null; // The reader instance of the given ReadableStream. @@ -92,87 +94,91 @@ export class Bundle { */ async getMetadata(): Promise { if (!this.metadata) { - const result = await this.nextElement(); - if (result === null || result instanceof SizedBundleElement) { - throw new Error(`The first element is not metadata, it is ${result}`); - } - this.metadata = (result as BundleElement).metadata; + await this.nextElement(); } return this.metadata!; } /** - * Asynchronously iterate through all bundle elements (except bundle metadata). + * Returns the next BundleElement (together with its byte size in the bundle) + * that has not been read from underlying ReadableStream. Returns null if we + * have reached the end of the stream. + * + * Throws an error if the first element is not a BundleMetadata. */ - async *elements(): AsyncIterableIterator { - let element = await this.nextElement(); - while (element !== null) { - if (element instanceof SizedBundleElement) { - yield element; + async nextElement(): Promise { + const element = await this.readNextElement(); + if (!element) { + return element; + } + + if (!this.metadata) { + if (element.isBundleMetadata()) { + this.metadata = element.payload.metadata; } else { - this.metadata = element.metadata; + this.raiseError( + `The first element of the bundle is not a metadata, it is ${JSON.stringify( + element.payload + )}` + ); } - element = await this.nextElement(); } + + return element; } - // Reads from the head of internal buffer, and pulling more data from underlying stream if a complete element - // cannot be found, until an element(including the prefixed length and the JSON string) is found. - // - // Once a complete element is read, it is dropped from internal buffer. - // - // Returns either the bundled element, or null if we have reached the end of the stream. - private async nextElement(): Promise< - BundleElement | SizedBundleElement | null - > { + /** + * Reads from the head of internal buffer, and pulling more data from underlying stream if a complete element + * cannot be found, until an element(including the prefixed length and the JSON string) is found. + * + * Once a complete element is read, it is dropped from internal buffer. + * + * Returns either the bundled element, or null if we have reached the end of the stream. + */ + private async readNextElement(): Promise { const lengthBuffer = await this.readLength(); if (lengthBuffer === null) { return null; } const lengthString = this.textDecoder.decode(lengthBuffer); - const length = parseInt(lengthString, 10); + const length = Number(lengthString); if (isNaN(length)) { - throw new Error(`length string (${lengthString}) is not valid number`); + this.raiseError(`length string (${lengthString}) is not valid number`); } const jsonString = await this.readJsonString(lengthBuffer.length, length); // Update the internal buffer to drop the read length and json string. this.buffer = this.buffer.slice(lengthBuffer.length + length); - if (!this.metadata) { - const element = JSON.parse(jsonString) as BundleElement; - return element; - } else { - return new SizedBundleElement( - JSON.parse(jsonString), - lengthBuffer.length + length - ); - } + return new SizedBundleElement( + JSON.parse(jsonString), + lengthBuffer.length + length + ); } // First index of '{' from the underlying buffer. private indexOfOpenBracket(): number { - return this.buffer.findIndex(v => v === 123); + return this.buffer.findIndex(v => v === '{'.charCodeAt(0)); } - // Reads from the beginning of the inernal buffer, until the first '{', and return + // Reads from the beginning of the internal buffer, until the first '{', and return // the content. // If reached end of the stream, returns a null. private async readLength(): Promise { let position = this.indexOfOpenBracket(); while (position < 0) { - const done = await this.pullMoreDataToBuffer(); - if (done) { + const bytesRead = await this.pullMoreDataToBuffer(); + if (bytesRead < 0) { if (this.buffer.length === 0) { return null; } position = this.indexOfOpenBracket(); // Underlying stream is closed, and we still cannot find a '{'. if (position < 0) { - throw new Error( - 'Reach to the end of bundle when a length string is expected.' + this.raiseError( + 'Reached the end of bundle when a length string is expected.' ); } } else { @@ -189,20 +195,28 @@ export class Bundle { // Returns a string decoded from the read bytes. private async readJsonString(start: number, length: number): Promise { while (this.buffer.length < start + length) { - const done = await this.pullMoreDataToBuffer(); - if (done) { - throw new Error('Reach to the end of bundle when more is expected.'); + const bytesRead = await this.pullMoreDataToBuffer(); + if (bytesRead < 0) { + this.raiseError('Reached the end of bundle when more is expected.'); } } return this.textDecoder.decode(this.buffer.slice(start, start + length)); } + private raiseError(message: string): void { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + this.reader.cancel('Invalid bundle format.'); + throw new Error(message); + } + // Pulls more data from underlying stream to internal buffer. // Returns a boolean indicating whether the stream is finished. - private async pullMoreDataToBuffer(): Promise { + private async pullMoreDataToBuffer(): Promise { const result = await this.reader.read(); + let bytesRead = -1; if (!result.done) { + bytesRead = result.value.length; const newBuffer = new Uint8Array( this.buffer.length + result.value.length ); @@ -210,6 +224,6 @@ export class Bundle { newBuffer.set(result.value, this.buffer.length); this.buffer = newBuffer; } - return result.done; + return bytesRead; } } diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index 39c705dc647..6e40459139a 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -15,7 +15,11 @@ * limitations under the License. */ import { expect } from 'chai'; -import { Bundle, toReadableStream } from '../../../src/util/bundle'; +import { + BundleReader, + SizedBundleElement, + toReadableStream +} from '../../../src/util/bundle_reader'; import { isNode } from '../../util/test_platform'; function readableStreamFromString( @@ -65,6 +69,7 @@ function lengthPrefixedString(o: {}): string { }); function genericBundleReadingTests(bytesPerRead: number): void { + const encoder = new TextEncoder(); // Setting up test data. const meta = { metadata: { @@ -160,13 +165,41 @@ function genericBundleReadingTests(bytesPerRead: number): void { }; const limitToLastQueryString = lengthPrefixedString(limitToLastQuery); - async function expectErrorFromBundle( + async function getAllElement( + bundle: BundleReader + ): Promise { + const result: SizedBundleElement[] = []; + while (true) { + const sizedElement = await bundle.nextElement(); + if (sizedElement === null) { + break; + } + if (!sizedElement.isBundleMetadata()) { + result.push(sizedElement); + } + } + + return Promise.resolve(result); + } + + function verifySizedElement( + element: SizedBundleElement, + payload: unknown, + payloadString: string + ): void { + expect(element.payload).to.deep.equal(payload); + expect(element.byteLength).to.equal( + encoder.encode(payloadString).byteLength + ); + } + + async function parseThroughBundle( bundleString: string, bytesPerRead: number, validMeta = false ): Promise { const bundleStream = readableStreamFromString(bundleString, bytesPerRead); - const bundle = new Bundle(bundleStream); + const bundle = new BundleReader(bundleStream); if (!validMeta) { await expect(await bundle.getMetadata()).should.be.rejected; @@ -174,10 +207,7 @@ function genericBundleReadingTests(bytesPerRead: number): void { expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); } - const actual = []; - for await (const sizedElement of bundle.elements()) { - actual.push(sizedElement); - } + await getAllElement(bundle); } it('reads with query and doc with bytesPerRead ' + bytesPerRead, async () => { @@ -189,31 +219,16 @@ function genericBundleReadingTests(bytesPerRead: number): void { doc1String, bytesPerRead ); - const bundle = new Bundle(bundleStream); + const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); - const actual = []; - for await (const sizedElement of bundle.elements()) { - actual.push(sizedElement); - } + const actual = await getAllElement(bundle); expect(actual.length).to.equal(4); - expect(actual[0]).to.deep.equal({ - payload: limitQuery, - byteLength: encoder.encode(limitQueryString).byteLength - }); - expect(actual[1]).to.deep.equal({ - payload: limitToLastQuery, - byteLength: encoder.encode(limitToLastQueryString).byteLength - }); - expect(actual[2]).to.deep.equal({ - payload: doc1Meta, - byteLength: encoder.encode(doc1MetaString).byteLength - }); - expect(actual[3]).to.deep.equal({ - payload: doc1, - byteLength: encoder.encode(doc1String).byteLength - }); + verifySizedElement(actual[0], limitQuery, limitQueryString); + verifySizedElement(actual[1], limitToLastQuery, limitToLastQueryString); + verifySizedElement(actual[2], doc1Meta, doc1MetaString); + verifySizedElement(actual[3], doc1, doc1String); }); it( @@ -228,33 +243,15 @@ function genericBundleReadingTests(bytesPerRead: number): void { doc2String, bytesPerRead ); - const bundle = new Bundle(bundleStream); + const bundle = new BundleReader(bundleStream); - const actual = []; - for await (const sizedElement of bundle.elements()) { - actual.push(sizedElement); - } + const actual = await getAllElement(bundle); expect(actual.length).to.equal(5); - expect(actual[0]).to.deep.equal({ - payload: doc1Meta, - byteLength: encoder.encode(doc1MetaString).byteLength - }); - expect(actual[1]).to.deep.equal({ - payload: doc1, - byteLength: encoder.encode(doc1String).byteLength - }); - expect(actual[2]).to.deep.equal({ - payload: limitQuery, - byteLength: encoder.encode(limitQueryString).byteLength - }); - expect(actual[3]).to.deep.equal({ - payload: doc2Meta, - byteLength: encoder.encode(doc2MetaString).byteLength - }); - expect(actual[4]).to.deep.equal({ - payload: doc2, - byteLength: encoder.encode(doc2String).byteLength - }); + verifySizedElement(actual[0], doc1Meta, doc1MetaString); + verifySizedElement(actual[1], doc1, doc1String); + verifySizedElement(actual[2], limitQuery, limitQueryString); + verifySizedElement(actual[3], doc2Meta, doc2MetaString); + verifySizedElement(actual[4], doc2, doc2String); // Reading metadata after other elements should also work. expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); @@ -268,23 +265,14 @@ function genericBundleReadingTests(bytesPerRead: number): void { metaString + doc1MetaString + doc1String, bytesPerRead ); - const bundle = new Bundle(bundleStream); + const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); - const actual = []; - for await (const sizedElement of bundle.elements()) { - actual.push(sizedElement); - } + const actual = await getAllElement(bundle); expect(actual.length).to.equal(2); - expect(actual[0]).to.deep.equal({ - payload: doc1Meta, - byteLength: encoder.encode(doc1MetaString).byteLength - }); - expect(actual[1]).to.deep.equal({ - payload: doc1, - byteLength: encoder.encode(doc1String).byteLength - }); + verifySizedElement(actual[0], doc1Meta, doc1MetaString); + verifySizedElement(actual[1], doc1, doc1String); } ); @@ -293,41 +281,26 @@ function genericBundleReadingTests(bytesPerRead: number): void { metaString + noDocMetaString + doc1MetaString + doc1String, bytesPerRead ); - const bundle = new Bundle(bundleStream); + const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); - const actual = []; - for await (const sizedElement of bundle.elements()) { - actual.push(sizedElement); - } + const actual = await getAllElement(bundle); expect(actual.length).to.equal(3); - expect(actual[0]).to.deep.equal({ - payload: noDocMeta, - byteLength: encoder.encode(noDocMetaString).byteLength - }); - expect(actual[1]).to.deep.equal({ - payload: doc1Meta, - byteLength: encoder.encode(doc1MetaString).byteLength - }); - expect(actual[2]).to.deep.equal({ - payload: doc1, - byteLength: encoder.encode(doc1String).byteLength - }); + verifySizedElement(actual[0], noDocMeta, noDocMetaString); + verifySizedElement(actual[1], doc1Meta, doc1MetaString); + verifySizedElement(actual[2], doc1, doc1String); }); it( 'reads without documents or query with bytesPerRead ' + bytesPerRead, async () => { const bundleStream = readableStreamFromString(metaString, bytesPerRead); - const bundle = new Bundle(bundleStream); + const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); - const actual = []; - for await (const sizedElement of bundle.elements()) { - actual.push(sizedElement); - } + const actual = await getAllElement(bundle); expect(actual.length).to.equal(0); } ); @@ -336,19 +309,24 @@ function genericBundleReadingTests(bytesPerRead: number): void { 'throws with ill-formatted bundle with bytesPerRead ' + bytesPerRead, async () => { await expect( - expectErrorFromBundle('metadata: "no length prefix"', bytesPerRead) + parseThroughBundle('metadata: "no length prefix"', bytesPerRead) ).to.be.rejected; await expect( - expectErrorFromBundle('{metadata: "no length prefix"}', bytesPerRead) + parseThroughBundle('{metadata: "no length prefix"}', bytesPerRead) ).to.be.rejected; await expect( - expectErrorFromBundle(metaString + 'invalid-string', bytesPerRead, true) + parseThroughBundle(metaString + 'invalid-string', bytesPerRead, true) ).to.be.rejected; - await expect(expectErrorFromBundle('1' + metaString, bytesPerRead)).to.be + await expect(parseThroughBundle('1' + metaString, bytesPerRead)).to.be .rejected; + + // First element is not BundleMetadata. + await expect( + parseThroughBundle(doc1MetaString + doc1String, bytesPerRead) + ).to.be.rejected; } ); } From 78248cd9b73b569d9cd1e90bfd8e226ff6c18499 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Sat, 23 May 2020 18:53:53 -0400 Subject: [PATCH 07/43] Support bundle reader for Node --- packages/firestore/src/platform/platform.ts | 32 ++++++++++++++++ .../src/platform_browser/browser_platform.ts | 27 ++++++++++++- .../src/platform_node/node_platform.ts | 13 ++++++- packages/firestore/src/util/bundle_reader.ts | 38 +++---------------- packages/firestore/test/util/test_platform.ts | 6 ++- 5 files changed, 81 insertions(+), 35 deletions(-) diff --git a/packages/firestore/src/platform/platform.ts b/packages/firestore/src/platform/platform.ts index 1ab582ea05f..f5c7e013713 100644 --- a/packages/firestore/src/platform/platform.ts +++ b/packages/firestore/src/platform/platform.ts @@ -50,6 +50,8 @@ export interface Platform { */ randomBytes(nBytes: number): Uint8Array; + toByteStreamReader(source: unknown): ByteStreamReader; + /** The Platform's 'window' implementation or null if not available. */ readonly window: Window | null; @@ -60,6 +62,36 @@ export interface Platform { readonly base64Available: boolean; } +export interface ByteStreamReader { + read(): Promise; +} + +export interface ByteStreamReadResult { + done: boolean; + value: Uint8Array; +} + +export function toByteStreamReader( + source: Uint8Array, + bytesPerRead = 10240 +): ByteStreamReader { + let readFrom = 0; + return new (class implements ByteStreamReader { + async read(): Promise { + if (readFrom < source.byteLength) { + const result = { + value: source.slice(readFrom, readFrom + bytesPerRead), + done: false + }; + readFrom += bytesPerRead; + return result; + } + + return { value: new Uint8Array(), done: true }; + } + })(); +} + /** * Provides singleton helpers where setup code can inject a platform at runtime. * setPlatform needs to be set before Firestore is used and must be set exactly diff --git a/packages/firestore/src/platform_browser/browser_platform.ts b/packages/firestore/src/platform_browser/browser_platform.ts index 38133db4df5..54be8b25cf5 100644 --- a/packages/firestore/src/platform_browser/browser_platform.ts +++ b/packages/firestore/src/platform_browser/browser_platform.ts @@ -16,7 +16,12 @@ */ import { DatabaseId, DatabaseInfo } from '../core/database_info'; -import { Platform } from '../platform/platform'; +import { + ByteStreamReader, + ByteStreamReadResult, + Platform, + toByteStreamReader +} from '../platform/platform'; import { Connection } from '../remote/connection'; import { JsonProtoSerializer } from '../remote/serializer'; import { ConnectivityMonitor } from './../remote/connectivity_monitor'; @@ -93,4 +98,24 @@ export class BrowserPlatform implements Platform { } return bytes; } + + toByteStreamReader(source: unknown): ByteStreamReader { + if (source instanceof Uint8Array) { + return toByteStreamReader(source); + } + if (source instanceof ArrayBuffer) { + return toByteStreamReader(new Uint8Array(source)); + } + if (source instanceof ReadableStream) { + const reader = source.getReader(); + return new (class implements ByteStreamReader { + read(): Promise { + return reader.read(); + } + })(); + } + throw new Error( + 'Source of `toByteStreamReader` has to be Uint8Array, ArrayBuffer or ReadableStream' + ); + } } diff --git a/packages/firestore/src/platform_node/node_platform.ts b/packages/firestore/src/platform_node/node_platform.ts index 2fe74f7d68b..c2a40b76aa9 100644 --- a/packages/firestore/src/platform_node/node_platform.ts +++ b/packages/firestore/src/platform_node/node_platform.ts @@ -19,7 +19,11 @@ import { randomBytes } from 'crypto'; import { inspect } from 'util'; import { DatabaseId, DatabaseInfo } from '../core/database_info'; -import { Platform } from '../platform/platform'; +import { + ByteStreamReader, + Platform, + toByteStreamReader +} from '../platform/platform'; import { Connection } from '../remote/connection'; import { JsonProtoSerializer } from '../remote/serializer'; import { Code, FirestoreError } from '../util/error'; @@ -83,4 +87,11 @@ export class NodePlatform implements Platform { return randomBytes(nBytes); } + + toByteStreamReader(source: unknown): ByteStreamReader { + if (source instanceof Uint8Array) { + return toByteStreamReader(source); + } + throw new Error('Source of `toByteStreamReader` has to be Uint8Array'); + } } diff --git a/packages/firestore/src/util/bundle_reader.ts b/packages/firestore/src/util/bundle_reader.ts index 8bfb19f8e5d..4b28467a706 100644 --- a/packages/firestore/src/util/bundle_reader.ts +++ b/packages/firestore/src/util/bundle_reader.ts @@ -19,6 +19,7 @@ import { BundleElement, BundleMetadata } from '../protos/firestore_bundle_proto'; +import { ByteStreamReader, PlatformSupport } from '../platform/platform'; /** * A complete element in the bundle stream, together with the byte length it @@ -36,29 +37,6 @@ export class SizedBundleElement { } } -/** - * Create a `ReadableStream` from a underlying buffer. - * - * @param data: Underlying buffer. - * @param bytesPerRead: How many bytes to read from the underlying buffer from each read through the stream. - */ -export function toReadableStream( - data: Uint8Array | ArrayBuffer, - bytesPerRead = 10240 -): ReadableStream { - let readFrom = 0; - return new ReadableStream({ - start(controller) {}, - async pull(controller): Promise { - controller.enqueue(data.slice(readFrom, readFrom + bytesPerRead)); - readFrom += bytesPerRead; - if (readFrom >= data.byteLength) { - controller.close(); - } - } - }); -} - /** * A class representing a bundle. * @@ -68,8 +46,8 @@ export function toReadableStream( export class BundleReader { // Cached bundle metadata. private metadata?: BundleMetadata | null; - // The reader instance of the given ReadableStream. - private reader: ReadableStreamDefaultReader; + // The reader to read from underlying binary bundle data source. + private reader: ByteStreamReader; // Internal buffer to hold bundle content, accumulating incomplete element content. private buffer: Uint8Array = new Uint8Array(); private textDecoder = new TextDecoder('utf-8'); @@ -80,13 +58,9 @@ export class BundleReader { | Uint8Array | ArrayBuffer ) { - if ( - bundleStream instanceof Uint8Array || - bundleStream instanceof ArrayBuffer - ) { - this.bundleStream = toReadableStream(bundleStream); - } - this.reader = (this.bundleStream as ReadableStream).getReader(); + this.reader = PlatformSupport.getPlatform().toByteStreamReader( + bundleStream + ); } /** diff --git a/packages/firestore/test/util/test_platform.ts b/packages/firestore/test/util/test_platform.ts index cd9d458f63d..b4c023d1907 100644 --- a/packages/firestore/test/util/test_platform.ts +++ b/packages/firestore/test/util/test_platform.ts @@ -16,7 +16,7 @@ */ import { DatabaseId, DatabaseInfo } from '../../src/core/database_info'; -import { Platform } from '../../src/platform/platform'; +import { ByteStreamReader, Platform } from '../../src/platform/platform'; import { Connection } from '../../src/remote/connection'; import { JsonProtoSerializer } from '../../src/remote/serializer'; import { debugAssert, fail } from '../../src/util/assert'; @@ -270,6 +270,10 @@ export class TestPlatform implements Platform { randomBytes(nBytes: number): Uint8Array { return this.basePlatform.randomBytes(nBytes); } + + toByteStreamReader(source: unknown): ByteStreamReader { + return this.basePlatform.toByteStreamReader(source); + } } /** Returns true if we are running under Node. */ From 83160a1f3b51b0cc19a0057b5d30109f9b6bf186 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 29 May 2020 14:29:05 -0400 Subject: [PATCH 08/43] Fix rebase errors. --- packages/firestore/src/platform/platform.ts | 3 + .../src/platform_browser/browser_platform.ts | 4 + .../firestore/test/unit/util/bundle.test.ts | 297 +++++++++--------- 3 files changed, 157 insertions(+), 147 deletions(-) diff --git a/packages/firestore/src/platform/platform.ts b/packages/firestore/src/platform/platform.ts index f5c7e013713..af830a468bd 100644 --- a/packages/firestore/src/platform/platform.ts +++ b/packages/firestore/src/platform/platform.ts @@ -64,6 +64,7 @@ export interface Platform { export interface ByteStreamReader { read(): Promise; + cancel(reason?: string): Promise; } export interface ByteStreamReadResult { @@ -89,6 +90,8 @@ export function toByteStreamReader( return { value: new Uint8Array(), done: true }; } + + async cancel(reason?: string): Promise {} })(); } diff --git a/packages/firestore/src/platform_browser/browser_platform.ts b/packages/firestore/src/platform_browser/browser_platform.ts index 54be8b25cf5..ff1ab177568 100644 --- a/packages/firestore/src/platform_browser/browser_platform.ts +++ b/packages/firestore/src/platform_browser/browser_platform.ts @@ -112,6 +112,10 @@ export class BrowserPlatform implements Platform { read(): Promise { return reader.read(); } + + cancel(reason?: string): Promise { + return reader.cancel(reason); + } })(); } throw new Error( diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index 6e40459139a..d68b739f77e 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -17,22 +17,32 @@ import { expect } from 'chai'; import { BundleReader, - SizedBundleElement, - toReadableStream + SizedBundleElement } from '../../../src/util/bundle_reader'; import { isNode } from '../../util/test_platform'; +/** + * Create a `ReadableStream` from a underlying buffer. + * + * @param data: Underlying buffer. + * @param bytesPerRead: How many bytes to read from the underlying buffer from each read through the stream. + */ function readableStreamFromString( content: string, bytesPerRead: number -): ReadableStream { - return toReadableStream(new TextEncoder().encode(content), bytesPerRead); -} - -function lengthPrefixedString(o: {}): string { - const str = JSON.stringify(o); - const l = new TextEncoder().encode(str).byteLength; - return `${l}${str}`; +): ReadableStream { + const data = new TextEncoder().encode(content); + let readFrom = 0; + return new ReadableStream({ + start(controller) {}, + async pull(controller): Promise { + controller.enqueue(data.slice(readFrom, readFrom + bytesPerRead)); + readFrom += bytesPerRead; + if (readFrom >= data.byteLength) { + controller.close(); + } + } + }); } // eslint-disable-next-line no-restricted-properties @@ -60,16 +70,82 @@ function lengthPrefixedString(o: {}): string { }); }); -// eslint-disable-next-line no-restricted-properties -(isNode() ? describe.skip : describe)('Bundle ', () => { - genericBundleReadingTests(1); - genericBundleReadingTests(4); - genericBundleReadingTests(64); - genericBundleReadingTests(1024); +describe.only('Bundle ', () => { + if (!isNode()) { + genericBundleReadingTests(1); + genericBundleReadingTests(4); + genericBundleReadingTests(64); + genericBundleReadingTests(1024); + } + genericBundleReadingTests(0); }); function genericBundleReadingTests(bytesPerRead: number): void { const encoder = new TextEncoder(); + + function testTextSuffix(): string { + if (bytesPerRead > 0) { + return ` from ReadableStream with bytesPerRead: ${bytesPerRead}`; + } + return ' from Uint8Array'; + } + + function bundleFromString(s: string): BundleReader { + if (bytesPerRead > 0) { + return new BundleReader(readableStreamFromString(s, bytesPerRead)); + } + return new BundleReader(encoder.encode(s)); + } + + function lengthPrefixedString(o: {}): string { + const str = JSON.stringify(o); + const l = new TextEncoder().encode(str).byteLength; + return `${l}${str}`; + } + + async function parseThroughBundle( + bundleString: string, + validMeta = false + ): Promise { + const bundle = bundleFromString(bundleString); + + if (!validMeta) { + await expect(await bundle.getMetadata()).should.be.rejected; + } else { + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + } + + await getAllElement(bundle); + } + + async function getAllElement( + bundle: BundleReader + ): Promise { + const result: SizedBundleElement[] = []; + while (true) { + const sizedElement = await bundle.nextElement(); + if (sizedElement === null) { + break; + } + if (!sizedElement.isBundleMetadata()) { + result.push(sizedElement); + } + } + + return Promise.resolve(result); + } + + function verifySizedElement( + element: SizedBundleElement, + payload: unknown, + payloadString: string + ): void { + expect(element.payload).to.deep.equal(payload); + expect(element.byteLength).to.equal( + encoder.encode(payloadString).byteLength + ); + } + // Setting up test data. const meta = { metadata: { @@ -165,61 +241,14 @@ function genericBundleReadingTests(bytesPerRead: number): void { }; const limitToLastQueryString = lengthPrefixedString(limitToLastQuery); - async function getAllElement( - bundle: BundleReader - ): Promise { - const result: SizedBundleElement[] = []; - while (true) { - const sizedElement = await bundle.nextElement(); - if (sizedElement === null) { - break; - } - if (!sizedElement.isBundleMetadata()) { - result.push(sizedElement); - } - } - - return Promise.resolve(result); - } - - function verifySizedElement( - element: SizedBundleElement, - payload: unknown, - payloadString: string - ): void { - expect(element.payload).to.deep.equal(payload); - expect(element.byteLength).to.equal( - encoder.encode(payloadString).byteLength - ); - } - - async function parseThroughBundle( - bundleString: string, - bytesPerRead: number, - validMeta = false - ): Promise { - const bundleStream = readableStreamFromString(bundleString, bytesPerRead); - const bundle = new BundleReader(bundleStream); - - if (!validMeta) { - await expect(await bundle.getMetadata()).should.be.rejected; - } else { - expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); - } - - await getAllElement(bundle); - } - - it('reads with query and doc with bytesPerRead ' + bytesPerRead, async () => { - const bundleStream = readableStreamFromString( + it('reads with query and doc' + testTextSuffix(), async () => { + const bundle = bundleFromString( metaString + limitQueryString + limitToLastQueryString + doc1MetaString + - doc1String, - bytesPerRead + doc1String ); - const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); @@ -231,57 +260,43 @@ function genericBundleReadingTests(bytesPerRead: number): void { verifySizedElement(actual[3], doc1, doc1String); }); - it( - 'reads with unexpected orders with bytesPerRead ' + bytesPerRead, - async () => { - const bundleStream = readableStreamFromString( - metaString + - doc1MetaString + - doc1String + - limitQueryString + - doc2MetaString + - doc2String, - bytesPerRead - ); - const bundle = new BundleReader(bundleStream); - - const actual = await getAllElement(bundle); - expect(actual.length).to.equal(5); - verifySizedElement(actual[0], doc1Meta, doc1MetaString); - verifySizedElement(actual[1], doc1, doc1String); - verifySizedElement(actual[2], limitQuery, limitQueryString); - verifySizedElement(actual[3], doc2Meta, doc2MetaString); - verifySizedElement(actual[4], doc2, doc2String); - - // Reading metadata after other elements should also work. - expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); - } - ); + it('reads with unexpected orders' + testTextSuffix(), async () => { + const bundle = bundleFromString( + metaString + + doc1MetaString + + doc1String + + limitQueryString + + doc2MetaString + + doc2String + ); - it( - 'reads without named query with bytesPerRead ' + bytesPerRead, - async () => { - const bundleStream = readableStreamFromString( - metaString + doc1MetaString + doc1String, - bytesPerRead - ); - const bundle = new BundleReader(bundleStream); + const actual = await getAllElement(bundle); + expect(actual.length).to.equal(5); + verifySizedElement(actual[0], doc1Meta, doc1MetaString); + verifySizedElement(actual[1], doc1, doc1String); + verifySizedElement(actual[2], limitQuery, limitQueryString); + verifySizedElement(actual[3], doc2Meta, doc2MetaString); + verifySizedElement(actual[4], doc2, doc2String); + + // Reading metadata after other elements should also work. + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + }); - expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + it('reads without named query' + testTextSuffix(), async () => { + const bundle = bundleFromString(metaString + doc1MetaString + doc1String); - const actual = await getAllElement(bundle); - expect(actual.length).to.equal(2); - verifySizedElement(actual[0], doc1Meta, doc1MetaString); - verifySizedElement(actual[1], doc1, doc1String); - } - ); + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + + const actual = await getAllElement(bundle); + expect(actual.length).to.equal(2); + verifySizedElement(actual[0], doc1Meta, doc1MetaString); + verifySizedElement(actual[1], doc1, doc1String); + }); - it('reads with deleted doc with bytesPerRead ' + bytesPerRead, async () => { - const bundleStream = readableStreamFromString( - metaString + noDocMetaString + doc1MetaString + doc1String, - bytesPerRead + it('reads with deleted doc' + testTextSuffix(), async () => { + const bundle = bundleFromString( + metaString + noDocMetaString + doc1MetaString + doc1String ); - const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); @@ -292,41 +307,29 @@ function genericBundleReadingTests(bytesPerRead: number): void { verifySizedElement(actual[2], doc1, doc1String); }); - it( - 'reads without documents or query with bytesPerRead ' + bytesPerRead, - async () => { - const bundleStream = readableStreamFromString(metaString, bytesPerRead); - const bundle = new BundleReader(bundleStream); + it('reads without documents or query' + testTextSuffix(), async () => { + const bundle = bundleFromString(metaString); - expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); - const actual = await getAllElement(bundle); - expect(actual.length).to.equal(0); - } - ); - - it( - 'throws with ill-formatted bundle with bytesPerRead ' + bytesPerRead, - async () => { - await expect( - parseThroughBundle('metadata: "no length prefix"', bytesPerRead) - ).to.be.rejected; - - await expect( - parseThroughBundle('{metadata: "no length prefix"}', bytesPerRead) - ).to.be.rejected; - - await expect( - parseThroughBundle(metaString + 'invalid-string', bytesPerRead, true) - ).to.be.rejected; - - await expect(parseThroughBundle('1' + metaString, bytesPerRead)).to.be - .rejected; - - // First element is not BundleMetadata. - await expect( - parseThroughBundle(doc1MetaString + doc1String, bytesPerRead) - ).to.be.rejected; - } - ); + const actual = await getAllElement(bundle); + expect(actual.length).to.equal(0); + }); + + it('throws with ill-formatted bundle' + testTextSuffix(), async () => { + await expect(parseThroughBundle('metadata: "no length prefix"')).to.be + .rejected; + + await expect(parseThroughBundle('{metadata: "no length prefix"}')).to.be + .rejected; + + await expect(parseThroughBundle(metaString + 'invalid-string', true)).to.be + .rejected; + + await expect(parseThroughBundle('1' + metaString)).to.be.rejected; + + // First element is not BundleMetadata. + await expect(parseThroughBundle(doc1MetaString + doc1String)).to.be + .rejected; + }); } From 24e10cb6e06f68c1e8e61859c4fe55c52778cc7f Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 29 May 2020 14:38:49 -0400 Subject: [PATCH 09/43] Remote 'only' --- packages/firestore/test/unit/util/bundle.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index d68b739f77e..ceeae8b2d76 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -70,7 +70,7 @@ function readableStreamFromString( }); }); -describe.only('Bundle ', () => { +describe('Bundle ', () => { if (!isNode()) { genericBundleReadingTests(1); genericBundleReadingTests(4); From 4cbe60899d2f19df04a9f5b0f0febd67273bbdd8 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 4 Jun 2020 22:16:35 -0400 Subject: [PATCH 10/43] Merge branch 'wuandy/Bundles' into wuandy/BundleReaderNode # Conflicts: # packages/firestore/src/util/bundle_reader.ts # packages/firestore/test/unit/util/bundle.test.ts --- packages/firestore/src/platform/platform.ts | 4 + packages/firestore/src/util/bundle_reader.ts | 39 ++------- .../firestore/test/unit/util/bundle.test.ts | 83 ++++++++++++------- 3 files changed, 64 insertions(+), 62 deletions(-) diff --git a/packages/firestore/src/platform/platform.ts b/packages/firestore/src/platform/platform.ts index af830a468bd..034e5a5e64b 100644 --- a/packages/firestore/src/platform/platform.ts +++ b/packages/firestore/src/platform/platform.ts @@ -109,6 +109,10 @@ export class PlatformSupport { PlatformSupport.platform = platform; } + private static _forceSetPlatform(platform: Platform): void { + PlatformSupport.platform = platform; + } + static getPlatform(): Platform { if (!PlatformSupport.platform) { fail('Platform not set'); diff --git a/packages/firestore/src/util/bundle_reader.ts b/packages/firestore/src/util/bundle_reader.ts index c8e7435953b..c608357a293 100644 --- a/packages/firestore/src/util/bundle_reader.ts +++ b/packages/firestore/src/util/bundle_reader.ts @@ -20,6 +20,7 @@ import { BundleMetadata } from '../protos/firestore_bundle_proto'; import { Deferred } from './promise'; +import { ByteStreamReader, PlatformSupport } from '../platform/platform'; /** * A complete element in the bundle stream, together with the byte length it @@ -37,30 +38,6 @@ export class SizedBundleElement { } } -/** - * Create a `ReadableStream` from a underlying buffer. - * - * @param data: Underlying buffer. - * @param bytesPerRead: How many bytes to read from the underlying buffer from - * each read through the stream. - */ -export function toReadableStream( - data: Uint8Array | ArrayBuffer, - bytesPerRead = 10240 -): ReadableStream { - let readFrom = 0; - return new ReadableStream({ - start(controller) {}, - async pull(controller): Promise { - controller.enqueue(data.slice(readFrom, readFrom + bytesPerRead)); - readFrom += bytesPerRead; - if (readFrom >= data.byteLength) { - controller.close(); - } - } - }); -} - /** * A class representing a bundle. * @@ -70,8 +47,8 @@ export function toReadableStream( export class BundleReader { /** Cached bundle metadata. */ private metadata: Deferred = new Deferred(); - /** The reader instance of the given ReadableStream. */ - private reader: ReadableStreamDefaultReader; + /** The reader to read from underlying binary bundle data source. */ + private reader: ByteStreamReader; /** * Internal buffer to hold bundle content, accumulating incomplete element * content. @@ -86,13 +63,9 @@ export class BundleReader { | Uint8Array | ArrayBuffer ) { - if ( - bundleStream instanceof Uint8Array || - bundleStream instanceof ArrayBuffer - ) { - this.bundleStream = toReadableStream(bundleStream); - } - this.reader = (this.bundleStream as ReadableStream).getReader(); + this.reader = PlatformSupport.getPlatform().toByteStreamReader( + bundleStream + ); // Read the metadata (which is the first element). this.nextElementImpl().then( diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index f163c29cbb5..42841c02746 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -17,17 +17,38 @@ import { expect } from 'chai'; import { BundleReader, - SizedBundleElement, - toReadableStream + SizedBundleElement } from '../../../src/util/bundle_reader'; -import { isNode } from '../../util/test_platform'; import { BundleElement } from '../../../src/protos/firestore_bundle_proto'; +import { isNode } from '../../util/test_platform'; +import { + PlatformSupport, + toByteStreamReader +} from '../../../src/platform/platform'; -function readableStreamFromString( +/** + * Create a `ReadableStream` from a string. + * + * @param content: Bundle in string. + * @param bytesPerRead: How many bytes to read from the underlying buffer from + * each read through the stream. + */ +export function readableStreamFromString( content: string, - bytesPerRead: number -): ReadableStream { - return toReadableStream(new TextEncoder().encode(content), bytesPerRead); + bytesPerRead = 10240 +): ReadableStream { + let readFrom = 0; + const data = new TextEncoder().encode(content); + return new ReadableStream({ + start(controller) {}, + async pull(controller): Promise { + controller.enqueue(data.slice(readFrom, readFrom + bytesPerRead)); + readFrom += bytesPerRead; + if (readFrom >= data.byteLength) { + controller.close(); + } + } + }); } function lengthPrefixedString(o: {}): string { @@ -60,7 +81,7 @@ describe('readableStreamFromString()', () => { }); }); -describe('Bundle ', () => { +describe.only('Bundle ', () => { genericBundleReadingTests(1); genericBundleReadingTests(4); genericBundleReadingTests(64); @@ -68,6 +89,22 @@ describe('Bundle ', () => { }); function genericBundleReadingTests(bytesPerRead: number): void { + if (isNode()) { + const platform = PlatformSupport.getPlatform(); + platform.toByteStreamReader = source => + toByteStreamReader(source as Uint8Array, bytesPerRead); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (PlatformSupport as any)._forceSetPlatform(platform); + } + + function bundleFromString(s: string): BundleReader { + if (isNode()) { + return new BundleReader(encoder.encode(s)); + } else { + return new BundleReader(readableStreamFromString(s, bytesPerRead)); + } + } + const encoder = new TextEncoder(); // Setting up test data. const meta: BundleElement = { @@ -197,8 +234,7 @@ function genericBundleReadingTests(bytesPerRead: number): void { bytesPerRead: number, validMeta = false ): Promise { - const bundleStream = readableStreamFromString(bundleString, bytesPerRead); - const bundle = new BundleReader(bundleStream); + const bundle = bundleFromString(bundleString); if (!validMeta) { await expect(await bundle.getMetadata()).should.be.rejected; @@ -210,15 +246,13 @@ function genericBundleReadingTests(bytesPerRead: number): void { } it('reads with query and doc with bytesPerRead ' + bytesPerRead, async () => { - const bundleStream = readableStreamFromString( + const bundle = bundleFromString( metaString + limitQueryString + limitToLastQueryString + doc1MetaString + - doc1String, - bytesPerRead + doc1String ); - const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); @@ -233,16 +267,14 @@ function genericBundleReadingTests(bytesPerRead: number): void { it( 'reads with unexpected orders with bytesPerRead ' + bytesPerRead, async () => { - const bundleStream = readableStreamFromString( + const bundle = bundleFromString( metaString + doc1MetaString + doc1String + limitQueryString + doc2MetaString + - doc2String, - bytesPerRead + doc2String ); - const bundle = new BundleReader(bundleStream); const actual = await getAllElements(bundle); expect(actual.length).to.equal(5); @@ -260,11 +292,7 @@ function genericBundleReadingTests(bytesPerRead: number): void { it( 'reads without named query with bytesPerRead ' + bytesPerRead, async () => { - const bundleStream = readableStreamFromString( - metaString + doc1MetaString + doc1String, - bytesPerRead - ); - const bundle = new BundleReader(bundleStream); + const bundle = bundleFromString(metaString + doc1MetaString + doc1String); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); @@ -276,11 +304,9 @@ function genericBundleReadingTests(bytesPerRead: number): void { ); it('reads with deleted doc with bytesPerRead ' + bytesPerRead, async () => { - const bundleStream = readableStreamFromString( - metaString + noDocMetaString + doc1MetaString + doc1String, - bytesPerRead + const bundle = bundleFromString( + metaString + noDocMetaString + doc1MetaString + doc1String ); - const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); @@ -294,8 +320,7 @@ function genericBundleReadingTests(bytesPerRead: number): void { it( 'reads without documents or query with bytesPerRead ' + bytesPerRead, async () => { - const bundleStream = readableStreamFromString(metaString, bytesPerRead); - const bundle = new BundleReader(bundleStream); + const bundle = bundleFromString(metaString); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); From 4313e51b4da4d04528764fe22965eced633b2764 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 5 Jun 2020 10:58:46 -0400 Subject: [PATCH 11/43] Added more comments, and more tests for Node. --- packages/firestore/src/platform/platform.ts | 25 ++++++++++++++++-- .../src/platform_browser/browser_platform.ts | 7 ++++- .../src/platform_node/node_platform.ts | 3 +++ packages/firestore/src/util/bundle_reader.ts | 6 +++-- .../test/unit/platform/platform.test.ts | 26 ++++++++++++++++++- .../firestore/test/unit/util/bundle.test.ts | 12 ++++++--- 6 files changed, 69 insertions(+), 10 deletions(-) diff --git a/packages/firestore/src/platform/platform.ts b/packages/firestore/src/platform/platform.ts index 034e5a5e64b..4bc25e25279 100644 --- a/packages/firestore/src/platform/platform.ts +++ b/packages/firestore/src/platform/platform.ts @@ -50,6 +50,9 @@ export interface Platform { */ randomBytes(nBytes: number): Uint8Array; + /** + * Builds a `ByteStreamReader` from a data source. + */ toByteStreamReader(source: unknown): ByteStreamReader; /** The Platform's 'window' implementation or null if not available. */ @@ -62,16 +65,31 @@ export interface Platform { readonly base64Available: boolean; } +/** + * An interface compatible with Web's ReadableStream.getReader() return type. + * + * This can be used as an abstraction to mimic `ReadableStream` where it is not + * available. + */ export interface ByteStreamReader { read(): Promise; cancel(reason?: string): Promise; } +/** + * An interface compatible with ReadableStreamReadResult. + */ export interface ByteStreamReadResult { done: boolean; - value: Uint8Array; + value?: Uint8Array; } +/** + * Builds a `ByteStreamReader` from a UInt8Array. + * @param source The data source to use. + * @param bytesPerRead How many bytes each `read()` from the returned reader + * will read. + */ export function toByteStreamReader( source: Uint8Array, bytesPerRead = 10240 @@ -88,7 +106,7 @@ export function toByteStreamReader( return result; } - return { value: new Uint8Array(), done: true }; + return { value: undefined, done: true }; } async cancel(reason?: string): Promise {} @@ -109,6 +127,9 @@ export class PlatformSupport { PlatformSupport.platform = platform; } + /** + * Forcing to set the platform instance, testing only! + */ private static _forceSetPlatform(platform: Platform): void { PlatformSupport.platform = platform; } diff --git a/packages/firestore/src/platform_browser/browser_platform.ts b/packages/firestore/src/platform_browser/browser_platform.ts index ff1ab177568..9941bcf4e7c 100644 --- a/packages/firestore/src/platform_browser/browser_platform.ts +++ b/packages/firestore/src/platform_browser/browser_platform.ts @@ -99,7 +99,12 @@ export class BrowserPlatform implements Platform { return bytes; } - toByteStreamReader(source: unknown): ByteStreamReader { + /** + * On web, a `ReadableStream` is wrapped around by a `ByteStreamReader`. + */ + toByteStreamReader( + source: Uint8Array | ArrayBuffer | ReadableStream + ): ByteStreamReader { if (source instanceof Uint8Array) { return toByteStreamReader(source); } diff --git a/packages/firestore/src/platform_node/node_platform.ts b/packages/firestore/src/platform_node/node_platform.ts index c2a40b76aa9..575de6616c2 100644 --- a/packages/firestore/src/platform_node/node_platform.ts +++ b/packages/firestore/src/platform_node/node_platform.ts @@ -88,6 +88,9 @@ export class NodePlatform implements Platform { return randomBytes(nBytes); } + /** + * On Node, only supported data source is a `Uint8Array` for now. + */ toByteStreamReader(source: unknown): ByteStreamReader { if (source instanceof Uint8Array) { return toByteStreamReader(source); diff --git a/packages/firestore/src/util/bundle_reader.ts b/packages/firestore/src/util/bundle_reader.ts index c608357a293..aff7a25c77d 100644 --- a/packages/firestore/src/util/bundle_reader.ts +++ b/packages/firestore/src/util/bundle_reader.ts @@ -21,6 +21,7 @@ import { } from '../protos/firestore_bundle_proto'; import { Deferred } from './promise'; import { ByteStreamReader, PlatformSupport } from '../platform/platform'; +import { debugAssert } from './assert'; /** * A complete element in the bundle stream, together with the byte length it @@ -204,11 +205,12 @@ export class BundleReader { private async pullMoreDataToBuffer(): Promise { const result = await this.reader.read(); if (!result.done) { + debugAssert(!!result.value, 'Read undefined when "done" is false.'); const newBuffer = new Uint8Array( - this.buffer.length + result.value.length + this.buffer.length + result.value!.length ); newBuffer.set(this.buffer); - newBuffer.set(result.value, this.buffer.length); + newBuffer.set(result.value!, this.buffer.length); this.buffer = newBuffer; } return result.done; diff --git a/packages/firestore/test/unit/platform/platform.test.ts b/packages/firestore/test/unit/platform/platform.test.ts index 289b0edee0d..631c49fb23d 100644 --- a/packages/firestore/test/unit/platform/platform.test.ts +++ b/packages/firestore/test/unit/platform/platform.test.ts @@ -16,10 +16,34 @@ */ import { expect } from 'chai'; -import { PlatformSupport } from '../../../src/platform/platform'; +import { + PlatformSupport, + toByteStreamReader +} from '../../../src/platform/platform'; describe('Platform', () => { it('can load the platform at runtime', () => { expect(PlatformSupport.getPlatform()).to.exist; }); + + it('toByteStreamReader() steps underlying data', async () => { + const encoder = new TextEncoder(); + const r = toByteStreamReader(encoder.encode('0123456789'), 4); + + let result = await r.read(); + expect(result.value).to.deep.equal(encoder.encode('0123')); + expect(result.done).to.be.false; + + result = await r.read(); + expect(result.value).to.deep.equal(encoder.encode('4567')); + expect(result.done).to.be.false; + + result = await r.read(); + expect(result.value).to.deep.equal(encoder.encode('89')); + expect(result.done).to.be.false; + + result = await r.read(); + expect(result.value).to.be.undefined; + expect(result.done).to.be.true; + }); }); diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index 42841c02746..e915d2e5dfc 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -57,11 +57,14 @@ function lengthPrefixedString(o: {}): string { return `${l}${str}`; } -describe('readableStreamFromString()', () => { +// Testing readableStreamFromString() is working as expected. +// eslint-disable-next-line no-restricted-properties +(isNode() ? describe.skip : describe)('readableStreamFromString()', () => { it('returns stepping readable stream', async () => { const encoder = new TextEncoder(); - const s = readableStreamFromString('0123456789', 4); - const r = s.getReader(); + const r = PlatformSupport.getPlatform().toByteStreamReader( + readableStreamFromString('0123456789', 4) + ); let result = await r.read(); expect(result.value).to.deep.equal(encoder.encode('0123')); @@ -81,7 +84,7 @@ describe('readableStreamFromString()', () => { }); }); -describe.only('Bundle ', () => { +describe('Bundle ', () => { genericBundleReadingTests(1); genericBundleReadingTests(4); genericBundleReadingTests(64); @@ -89,6 +92,7 @@ describe.only('Bundle ', () => { }); function genericBundleReadingTests(bytesPerRead: number): void { + // On Node, we need to override `bytesPerRead` from it's platform's `toByteStreamReader` call. if (isNode()) { const platform = PlatformSupport.getPlatform(); platform.toByteStreamReader = source => From 296cfc4300347eb08adc1e232fd792f8370fc462 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 5 Jun 2020 11:15:01 -0400 Subject: [PATCH 12/43] Implement BundleCache. --- packages/firestore/src/core/bundle.ts | 39 ++++ .../firestore/src/core/component_provider.ts | 3 +- packages/firestore/src/local/bundle_cache.ts | 60 +++++ .../src/local/indexeddb_bundle_cache.ts | 105 +++++++++ .../src/local/indexeddb_persistence.ts | 11 + .../firestore/src/local/indexeddb_schema.ts | 18 +- .../firestore/src/local/local_serializer.ts | 80 +++++++ .../src/local/memory_bundle_cache.ts | 66 ++++++ .../firestore/src/local/memory_persistence.ts | 14 +- packages/firestore/src/local/persistence.ts | 10 + packages/firestore/src/remote/serializer.ts | 8 +- .../test/unit/local/bundle_cache.test.ts | 207 ++++++++++++++++++ .../unit/local/persistence_test_helpers.ts | 9 +- .../test/unit/local/test_bundle_cache.ts | 73 ++++++ .../test/unit/specs/spec_test_components.ts | 6 +- 15 files changed, 692 insertions(+), 17 deletions(-) create mode 100644 packages/firestore/src/core/bundle.ts create mode 100644 packages/firestore/src/local/bundle_cache.ts create mode 100644 packages/firestore/src/local/indexeddb_bundle_cache.ts create mode 100644 packages/firestore/src/local/memory_bundle_cache.ts create mode 100644 packages/firestore/test/unit/local/bundle_cache.test.ts create mode 100644 packages/firestore/test/unit/local/test_bundle_cache.ts diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts new file mode 100644 index 00000000000..0571664b7f4 --- /dev/null +++ b/packages/firestore/src/core/bundle.ts @@ -0,0 +1,39 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Query } from './query'; +import { SnapshotVersion } from './snapshot_version'; + +/** + * Represents a Firestore bundle saved by the SDK in its local storage. + */ +export interface Bundle { + readonly id: string; + readonly version: number; + // When the saved bundle is built from the server SDKs. + readonly createTime: SnapshotVersion; +} + +/** + * Represents a Query saved by the SDK in its local storage. + */ +export interface NamedQuery { + readonly name: string; + readonly query: Query; + // When the results for this query are read to the saved bundle. + readonly readTime: SnapshotVersion; +} diff --git a/packages/firestore/src/core/component_provider.ts b/packages/firestore/src/core/component_provider.ts index 2c34723ecef..38ba5d2c763 100644 --- a/packages/firestore/src/core/component_provider.ts +++ b/packages/firestore/src/core/component_provider.ts @@ -137,7 +137,8 @@ export class MemoryComponentProvider implements ComponentProvider { !cfg.persistenceSettings.durable, 'Can only start memory persistence' ); - return new MemoryPersistence(MemoryEagerDelegate.factory); + const serializer = cfg.platform.newSerializer(cfg.databaseInfo.databaseId); + return new MemoryPersistence(MemoryEagerDelegate.factory, serializer); } createRemoteStore(cfg: ComponentConfiguration): RemoteStore { diff --git a/packages/firestore/src/local/bundle_cache.ts b/packages/firestore/src/local/bundle_cache.ts new file mode 100644 index 00000000000..e24d6ebc52f --- /dev/null +++ b/packages/firestore/src/local/bundle_cache.ts @@ -0,0 +1,60 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PersistenceTransaction } from './persistence'; +import { PersistencePromise } from './persistence_promise'; +import * as bundleProto from '../protos/firestore_bundle_proto'; +import { Bundle, NamedQuery } from '../core/bundle'; + +/** + * Provides interfaces to save and read Firestore bundles. + */ +export interface BundleCache { + /** + * Gets a saved `Bundle` for a given `bundleId`, returns undefined if + * no bundles are found under the given id. + */ + getBundle( + transaction: PersistenceTransaction, + bundleId: string + ): PersistencePromise; + + /** + * Saves a `BundleMetadata` from a bundle into local storage, using its id as the persistent key. + */ + saveBundleMetadata( + transaction: PersistenceTransaction, + metadata: bundleProto.BundleMetadata + ): PersistencePromise; + + /** + * Gets a saved `NamedQuery` for the given query name. Returns undefined if + * no queries are found under the given name. + */ + getNamedQuery( + transaction: PersistenceTransaction, + queryName: string + ): PersistencePromise; + + /** + * Saves a `NamedQuery` from a bundle, using its name as the persistent key. + */ + saveNamedQuery( + transaction: PersistenceTransaction, + query: bundleProto.NamedQuery + ): PersistencePromise; +} diff --git a/packages/firestore/src/local/indexeddb_bundle_cache.ts b/packages/firestore/src/local/indexeddb_bundle_cache.ts new file mode 100644 index 00000000000..a23546e3d4b --- /dev/null +++ b/packages/firestore/src/local/indexeddb_bundle_cache.ts @@ -0,0 +1,105 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PersistenceTransaction } from './persistence'; +import { PersistencePromise } from './persistence_promise'; +import * as bundleProto from '../protos/firestore_bundle_proto'; +import { BundleCache } from './bundle_cache'; +import { + DbBundle, + DbBundlesKey, + DbNamedQuery, + DbNamedQueriesKey +} from './indexeddb_schema'; +import { SimpleDbStore } from './simple_db'; +import { IndexedDbPersistence } from './indexeddb_persistence'; +import { LocalSerializer } from './local_serializer'; +import { Bundle, NamedQuery } from '../core/bundle'; + +export class IndexedDbBundleCache implements BundleCache { + constructor(private serializer: LocalSerializer) {} + + getBundle( + transaction: PersistenceTransaction, + bundleId: string + ): PersistencePromise { + return bundlesStore(transaction) + .get(bundleId) + .next(bundle => { + if (bundle) { + return this.serializer.fromDbBundle(bundle!); + } + return undefined; + }); + } + + saveBundleMetadata( + transaction: PersistenceTransaction, + bundleMetadata: bundleProto.BundleMetadata + ): PersistencePromise { + return bundlesStore(transaction).put( + this.serializer.toDbBundle(bundleMetadata) + ); + } + + getNamedQuery( + transaction: PersistenceTransaction, + queryName: string + ): PersistencePromise { + return namedQueriesStore(transaction) + .get(queryName) + .next(query => { + if (query) { + return this.serializer.fromDbNamedQuery(query!); + } + return undefined; + }); + } + + saveNamedQuery( + transaction: PersistenceTransaction, + query: bundleProto.NamedQuery + ): PersistencePromise { + return namedQueriesStore(transaction).put( + this.serializer.toDbNamedQuery(query) + ); + } +} + +/** + * Helper to get a typed SimpleDbStore for the bundles object store. + */ +function bundlesStore( + txn: PersistenceTransaction +): SimpleDbStore { + return IndexedDbPersistence.getStore( + txn, + DbBundle.store + ); +} + +/** + * Helper to get a typed SimpleDbStore for the namedQueries object store. + */ +function namedQueriesStore( + txn: PersistenceTransaction +): SimpleDbStore { + return IndexedDbPersistence.getStore( + txn, + DbNamedQuery.store + ); +} diff --git a/packages/firestore/src/local/indexeddb_persistence.ts b/packages/firestore/src/local/indexeddb_persistence.ts index 4409e0fab79..86d183f8f2e 100644 --- a/packages/firestore/src/local/indexeddb_persistence.ts +++ b/packages/firestore/src/local/indexeddb_persistence.ts @@ -32,6 +32,7 @@ import { EncodedResourcePath, encodeResourcePath } from './encoded_resource_path'; +import { IndexedDbBundleCache } from './indexeddb_bundle_cache'; import { IndexedDbIndexManager } from './indexeddb_index_manager'; import { IndexedDbMutationQueue, @@ -226,6 +227,7 @@ export class IndexedDbPersistence implements Persistence { private readonly targetCache: IndexedDbTargetCache; private readonly indexManager: IndexedDbIndexManager; private readonly remoteDocumentCache: IndexedDbRemoteDocumentCache; + private readonly bundleCache: IndexedDbBundleCache; private readonly webStorage: Storage; readonly referenceDelegate: IndexedDbLruDelegate; @@ -259,6 +261,7 @@ export class IndexedDbPersistence implements Persistence { this.serializer, this.indexManager ); + this.bundleCache = new IndexedDbBundleCache(this.serializer); if (platform.window && platform.window.localStorage) { this.window = platform.window; this.webStorage = this.window.localStorage; @@ -763,6 +766,14 @@ export class IndexedDbPersistence implements Persistence { return this.indexManager; } + getBundleCache(): IndexedDbBundleCache { + debugAssert( + this.started, + 'Cannot initialize BundleCache before persistence is started.' + ); + return this.bundleCache; + } + runTransaction( action: string, mode: PersistenceTransactionMode, diff --git a/packages/firestore/src/local/indexeddb_schema.ts b/packages/firestore/src/local/indexeddb_schema.ts index 2026b373adb..579a8d9eeeb 100644 --- a/packages/firestore/src/local/indexeddb_schema.ts +++ b/packages/firestore/src/local/indexeddb_schema.ts @@ -1092,11 +1092,11 @@ export type DbBundlesKey = string; /** * A object representing a bundle loaded by the SDK. */ -export class DbBundles { +export class DbBundle { /** Name of the IndexedDb object store. */ static store = 'bundles'; - static keyPath = ['bundleId']; + static keyPath = 'bundleId'; constructor( /** The ID of the loaded bundle. */ @@ -1109,8 +1109,8 @@ export class DbBundles { } function createBundlesStore(db: IDBDatabase): void { - db.createObjectStore(DbBundles.store, { - keyPath: DbBundles.keyPath + db.createObjectStore(DbBundle.store, { + keyPath: DbBundle.keyPath }); } @@ -1119,11 +1119,11 @@ export type DbNamedQueriesKey = string; /** * A object representing a named query loaded by the SDK via a bundle. */ -export class DbNamedQueries { +export class DbNamedQuery { /** Name of the IndexedDb object store. */ static store = 'namedQueries'; - static keyPath = ['name']; + static keyPath = 'name'; constructor( /** The name of the query. */ @@ -1136,8 +1136,8 @@ export class DbNamedQueries { } function createNamedQueriesStore(db: IDBDatabase): void { - db.createObjectStore(DbNamedQueries.store, { - keyPath: DbNamedQueries.keyPath + db.createObjectStore(DbNamedQuery.store, { + keyPath: DbNamedQuery.keyPath }); } @@ -1174,7 +1174,7 @@ export const V8_STORES = [...V6_STORES, DbCollectionParent.store]; // V10 does not change the set of stores. -export const V11_STORES = [...V8_STORES, DbCollectionParent.store]; +export const V11_STORES = [...V8_STORES, DbBundle.store, DbNamedQuery.store]; /** * The list of all default IndexedDB stores used throughout the SDK. This is diff --git a/packages/firestore/src/local/local_serializer.ts b/packages/firestore/src/local/local_serializer.ts index e15697b934c..a6176ceb2e1 100644 --- a/packages/firestore/src/local/local_serializer.ts +++ b/packages/firestore/src/local/local_serializer.ts @@ -25,13 +25,16 @@ import { } from '../model/document'; import { DocumentKey } from '../model/document_key'; import { MutationBatch } from '../model/mutation_batch'; +import * as bundleProto from '../protos/firestore_bundle_proto'; import * as api from '../protos/firestore_proto_api'; import { JsonProtoSerializer } from '../remote/serializer'; import { debugAssert, fail } from '../util/assert'; import { ByteString } from '../util/byte_string'; import { Target } from '../core/target'; import { + DbBundle, DbMutationBatch, + DbNamedQuery, DbNoDocument, DbQuery, DbRemoteDocument, @@ -41,6 +44,8 @@ import { DbUnknownDocument } from './indexeddb_schema'; import { TargetData, TargetPurpose } from './target_data'; +import { Bundle, NamedQuery } from '../core/bundle'; +import { Query } from '../core/query'; /** Serializer for values stored in the LocalStore. */ export class LocalSerializer { @@ -231,6 +236,81 @@ export class LocalSerializer { queryProto ); } + + /** Encodes a DbBundle to a Bundle. */ + fromDbBundle(dbBundle: DbBundle): Bundle { + return { + id: dbBundle.bundleId, + createTime: this.fromDbTimestamp(dbBundle.createTime), + version: dbBundle.version + }; + } + + /** Encodes a BundleMetadata to a DbBundle. */ + toDbBundle(metadata: bundleProto.BundleMetadata): DbBundle { + return { + bundleId: metadata.id!, + createTime: this.toDbTimestamp( + this.remoteSerializer.fromVersion(metadata.createTime!) + ), + version: metadata.version! + }; + } + + /** Encodes a DbNamedQuery to a NamedQuery. */ + fromDbNamedQuery(dbNamedQuery: DbNamedQuery): NamedQuery { + return { + name: dbNamedQuery.name, + query: this.fromBundledQuery(dbNamedQuery.bundledQuery), + readTime: this.fromDbTimestamp(dbNamedQuery.readTime) + }; + } + + /** Encodes a NamedQuery from bundle proto to a DbNamedQuery. */ + toDbNamedQuery(query: bundleProto.NamedQuery): DbNamedQuery { + return { + name: query.name!, + readTime: this.toDbTimestamp( + this.remoteSerializer.fromVersion(query.readTime!) + ), + bundledQuery: query.bundledQuery! + }; + } + + /** + * Encodes a `BundledQuery` from bundle proto to a Query object. + * + * This reconstructs the original query used to build the bundle being loaded, + * including features exists only in SDKs (for example: limit-to-last). + */ + fromBundledQuery(bundledQuery: bundleProto.BundledQuery): Query { + const query = this.remoteSerializer.convertQueryTargetToQuery({ + parent: bundledQuery.parent!, + structuredQuery: bundledQuery.structuredQuery! + }); + if (bundledQuery.limitType === 'LAST') { + return query.withLimitToLast(query.limit); + } + return query; + } + + /** Encodes a NamedQuery proto object to a NamedQuery model object. */ + fromProtoNamedQuery(namedQuery: bundleProto.NamedQuery): NamedQuery { + return { + name: namedQuery.name!, + query: this.fromBundledQuery(namedQuery.bundledQuery!), + readTime: this.remoteSerializer.fromVersion(namedQuery.readTime!) + }; + } + + /** Encodes a BundleMetadata proto object to a Bundle model object. */ + fromBundleMetadata(metadata: bundleProto.BundleMetadata): Bundle { + return { + id: metadata.id!, + version: metadata.version!, + createTime: this.remoteSerializer.fromVersion(metadata.createTime!) + }; + } } /** diff --git a/packages/firestore/src/local/memory_bundle_cache.ts b/packages/firestore/src/local/memory_bundle_cache.ts new file mode 100644 index 00000000000..64cbd7ef726 --- /dev/null +++ b/packages/firestore/src/local/memory_bundle_cache.ts @@ -0,0 +1,66 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PersistenceTransaction } from './persistence'; +import { PersistencePromise } from './persistence_promise'; +import * as bundleProto from '../protos/firestore_bundle_proto'; +import { BundleCache } from './bundle_cache'; +import { Bundle, NamedQuery } from '../core/bundle'; +import { LocalSerializer } from './local_serializer'; + +export class MemoryBundleCache implements BundleCache { + private bundles = new Map(); + private namedQueries = new Map(); + + constructor(private serializer: LocalSerializer) {} + + getBundle( + transaction: PersistenceTransaction, + bundleId: string + ): PersistencePromise { + return PersistencePromise.resolve(this.bundles.get(bundleId)); + } + + saveBundleMetadata( + transaction: PersistenceTransaction, + bundleMetadata: bundleProto.BundleMetadata + ): PersistencePromise { + this.bundles.set( + bundleMetadata.id!, + this.serializer.fromBundleMetadata(bundleMetadata) + ); + return PersistencePromise.resolve(); + } + + getNamedQuery( + transaction: PersistenceTransaction, + queryName: string + ): PersistencePromise { + return PersistencePromise.resolve(this.namedQueries.get(queryName)); + } + + saveNamedQuery( + transaction: PersistenceTransaction, + query: bundleProto.NamedQuery + ): PersistencePromise { + this.namedQueries.set( + query.name!, + this.serializer.fromProtoNamedQuery(query) + ); + return PersistencePromise.resolve(); + } +} diff --git a/packages/firestore/src/local/memory_persistence.ts b/packages/firestore/src/local/memory_persistence.ts index 391f8686565..ce84ff95ff2 100644 --- a/packages/firestore/src/local/memory_persistence.ts +++ b/packages/firestore/src/local/memory_persistence.ts @@ -45,6 +45,9 @@ import { import { PersistencePromise } from './persistence_promise'; import { ReferenceSet } from './reference_set'; import { TargetData } from './target_data'; +import { MemoryBundleCache } from './memory_bundle_cache'; +import { JsonProtoSerializer } from '../remote/serializer'; +import { LocalSerializer } from './local_serializer'; const LOG_TAG = 'MemoryPersistence'; /** @@ -63,7 +66,9 @@ export class MemoryPersistence implements Persistence { private mutationQueues: { [user: string]: MemoryMutationQueue } = {}; private readonly remoteDocumentCache: MemoryRemoteDocumentCache; private readonly targetCache: MemoryTargetCache; + private readonly bundleCache: MemoryBundleCache; private readonly listenSequence = new ListenSequence(0); + private serializer: LocalSerializer; private _started = false; @@ -76,7 +81,8 @@ export class MemoryPersistence implements Persistence { * checked or asserted on every access. */ constructor( - referenceDelegateFactory: (p: MemoryPersistence) => MemoryReferenceDelegate + referenceDelegateFactory: (p: MemoryPersistence) => MemoryReferenceDelegate, + serializer: JsonProtoSerializer ) { this._started = true; this.referenceDelegate = referenceDelegateFactory(this); @@ -88,6 +94,8 @@ export class MemoryPersistence implements Persistence { this.indexManager, sizer ); + this.serializer = new LocalSerializer(serializer); + this.bundleCache = new MemoryBundleCache(this.serializer); } start(): Promise { @@ -132,6 +140,10 @@ export class MemoryPersistence implements Persistence { return this.remoteDocumentCache; } + getBundleCache(): MemoryBundleCache { + return this.bundleCache; + } + runTransaction( action: string, mode: PersistenceTransactionMode, diff --git a/packages/firestore/src/local/persistence.ts b/packages/firestore/src/local/persistence.ts index 146b0331738..84debbc8afa 100644 --- a/packages/firestore/src/local/persistence.ts +++ b/packages/firestore/src/local/persistence.ts @@ -25,6 +25,7 @@ import { PersistencePromise } from './persistence_promise'; import { TargetCache } from './target_cache'; import { RemoteDocumentCache } from './remote_document_cache'; import { TargetData } from './target_data'; +import { BundleCache } from './bundle_cache'; export const PRIMARY_LEASE_LOST_ERROR_MSG = 'The current tab is not in the required state to perform this operation. ' + @@ -216,6 +217,15 @@ export interface Persistence { */ getRemoteDocumentCache(): RemoteDocumentCache; + /** + * Returns a BundleCache representing the persisted cache of loaded bundles. + * + * Note: The implementation is free to return the same instance every time + * this is called. In particular, the memory-backed implementation does this + * to emulate the persisted implementation to the extent possible. + */ + getBundleCache(): BundleCache; + /** * Returns an IndexManager instance that manages our persisted query indexes. * diff --git a/packages/firestore/src/remote/serializer.ts b/packages/firestore/src/remote/serializer.ts index de29942ffc2..db1ff3f3c39 100644 --- a/packages/firestore/src/remote/serializer.ts +++ b/packages/firestore/src/remote/serializer.ts @@ -787,7 +787,7 @@ export class JsonProtoSerializer { return result; } - fromQueryTarget(target: api.QueryTarget): Target { + convertQueryTargetToQuery(target: api.QueryTarget): Query { let path = this.fromQueryPath(target.parent!); const query = target.structuredQuery!; @@ -840,7 +840,11 @@ export class JsonProtoSerializer { LimitType.First, startAt, endAt - ).toTarget(); + ); + } + + fromQueryTarget(target: api.QueryTarget): Target { + return this.convertQueryTargetToQuery(target).toTarget(); } toListenRequestLabels( diff --git a/packages/firestore/test/unit/local/bundle_cache.test.ts b/packages/firestore/test/unit/local/bundle_cache.test.ts new file mode 100644 index 00000000000..11c7b1c7e67 --- /dev/null +++ b/packages/firestore/test/unit/local/bundle_cache.test.ts @@ -0,0 +1,207 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; +import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; +import { filter, orderBy, path } from '../../util/helpers'; +import * as persistenceHelpers from './persistence_test_helpers'; +import { TestBundleCache } from './test_bundle_cache'; +import { SnapshotVersion } from '../../../src/core/snapshot_version'; +import { Timestamp } from '../../../src/api/timestamp'; +import { Query } from '../../../src/core/query'; +import { JSON_SERIALIZER } from './persistence_test_helpers'; +import { ResourcePath } from '../../../src/model/path'; +import { NamedQuery } from '../../../src/core/bundle'; + +describe('MemoryBundleCache', () => { + let cache: TestBundleCache; + + beforeEach(async () => { + cache = await persistenceHelpers + .testMemoryEagerPersistence() + .then(persistence => new TestBundleCache(persistence)); + }); + + genericBundleCacheTests(() => cache); +}); + +describe('IndexedDbBundleCache', () => { + if (!IndexedDbPersistence.isAvailable()) { + console.warn('No IndexedDB. Skipping IndexedDbBundleCache tests.'); + return; + } + + let cache: TestBundleCache; + let persistence: IndexedDbPersistence; + beforeEach(async () => { + persistence = await persistenceHelpers.testIndexedDbPersistence({ + synchronizeTabs: true + }); + cache = new TestBundleCache(persistence); + }); + + afterEach(async () => { + await persistence.shutdown(); + await persistenceHelpers.clearTestPersistence(); + }); + + genericBundleCacheTests(() => cache); +}); + +/** + * Defines the set of tests to run against both bundle cache implementations. + */ +function genericBundleCacheTests(cacheFn: () => TestBundleCache): void { + let cache: TestBundleCache; + + beforeEach(async () => { + cache = cacheFn(); + }); + + function verifyNamedQuery( + actual: NamedQuery, + expectedName: string, + expectedQuery: Query, + expectedReadSeconds: number, + expectedReadNanos: number + ):void { + expect(actual.name).to.equal(expectedName); + expect(actual.query.isEqual(expectedQuery)).to.be.true; + expect( + actual.readTime.isEqual( + SnapshotVersion.fromTimestamp( + new Timestamp(expectedReadSeconds, expectedReadNanos) + ) + ) + ).to.be.true; + } + + it('returns undefined when bundle id is not found', async () => { + expect(await cache.getBundle('bundle-1')).to.be.undefined; + }); + + it('returns saved bundle', async () => { + await cache.saveBundleMetadata({ + id: 'bundle-1', + version: 1, + createTime: { seconds: 1, nanos: 9999 } + }); + expect(await cache.getBundle('bundle-1')).to.deep.equal({ + id: 'bundle-1', + version: 1, + createTime: SnapshotVersion.fromTimestamp(new Timestamp(1, 9999)) + }); + + // Overwrite + await cache.saveBundleMetadata({ + id: 'bundle-1', + version: 2, + createTime: { seconds: 2, nanos: 1111 } + }); + expect(await cache.getBundle('bundle-1')).to.deep.equal({ + id: 'bundle-1', + version: 2, + createTime: SnapshotVersion.fromTimestamp(new Timestamp(2, 1111)) + }); + }); + + it('returns undefined when query name is not found', async () => { + expect(await cache.getNamedQuery('query-1')).to.be.undefined; + }); + + it('returns saved collection queries', async () => { + const query = Query.atPath(path('collection')) + .addFilter(filter('sort', '>=', 2)) + .addOrderBy(orderBy('sort')); + const queryTarget = JSON_SERIALIZER.toQueryTarget(query.toTarget()); + + await cache.setNamedQuery({ + name: 'query-1', + readTime: { seconds: 1, nanos: 9999 }, + bundledQuery: { + parent: queryTarget.parent, + structuredQuery: queryTarget.structuredQuery + } + }); + + const namedQuery = await cache.getNamedQuery('query-1'); + verifyNamedQuery(namedQuery!, 'query-1', query, 1, 9999); + }); + + it('returns saved collection group queries', async () => { + const query = new Query(ResourcePath.EMPTY_PATH, 'collection'); + const queryTarget = JSON_SERIALIZER.toQueryTarget(query.toTarget()); + + await cache.setNamedQuery({ + name: 'query-1', + readTime: { seconds: 1, nanos: 9999 }, + bundledQuery: { + parent: queryTarget.parent, + structuredQuery: queryTarget.structuredQuery, + limitType: undefined + } + }); + + const namedQuery = await cache.getNamedQuery('query-1'); + verifyNamedQuery(namedQuery!, 'query-1', query, 1, 9999); + }); + + it('returns expected limit queries', async () => { + const query = Query.atPath(path('collection')) + .addOrderBy(orderBy('sort')) + .withLimitToFirst(3); + const queryTarget = JSON_SERIALIZER.toQueryTarget(query.toTarget()); + + await cache.setNamedQuery({ + name: 'query-1', + readTime: { seconds: 1, nanos: 9999 }, + bundledQuery: { + parent: queryTarget.parent, + structuredQuery: queryTarget.structuredQuery, + limitType: 'FIRST' + } + }); + + const namedQuery = await cache.getNamedQuery('query-1'); + verifyNamedQuery(namedQuery!, 'query-1', query, 1, 9999); + }); + + it('returns expected limit to last queries', async () => { + const query = Query.atPath(path('collection')) + .addOrderBy(orderBy('sort')) + .withLimitToLast(3); + // Simulating bundle building for limit-to-last queries from the server + // SDKs: they save the equivelent limit-to-first queries with a limitType + // value 'LAST'. Client SDKs should apply a withLimitToLast when they see + // limitType 'LAST' from bundles. + const limitQuery = query.withLimitToFirst(3); + const queryTarget = JSON_SERIALIZER.toQueryTarget(limitQuery.toTarget()); + + await cache.setNamedQuery({ + name: 'query-1', + readTime: { seconds: 1, nanos: 9999 }, + bundledQuery: { + parent: queryTarget.parent, + structuredQuery: queryTarget.structuredQuery, + limitType: 'LAST' + } + }); + + const namedQuery = await cache.getNamedQuery('query-1'); + verifyNamedQuery(namedQuery!, 'query-1', query, 1, 9999); + }); +} diff --git a/packages/firestore/test/unit/local/persistence_test_helpers.ts b/packages/firestore/test/unit/local/persistence_test_helpers.ts index b60b32d4264..9733ae9e009 100644 --- a/packages/firestore/test/unit/local/persistence_test_helpers.ts +++ b/packages/firestore/test/unit/local/persistence_test_helpers.ts @@ -86,7 +86,7 @@ export const INDEXEDDB_TEST_DATABASE_NAME = IndexedDbPersistence.buildStoragePrefix(TEST_DATABASE_INFO) + IndexedDbPersistence.MAIN_DATABASE; -const JSON_SERIALIZER = new JsonProtoSerializer(TEST_DATABASE_ID, { +export const JSON_SERIALIZER = new JsonProtoSerializer(TEST_DATABASE_ID, { useProto3Json: true }); @@ -131,13 +131,16 @@ export async function testIndexedDbPersistence( /** Creates and starts a MemoryPersistence instance for testing. */ export async function testMemoryEagerPersistence(): Promise { - return new MemoryPersistence(MemoryEagerDelegate.factory); + return new MemoryPersistence(MemoryEagerDelegate.factory, JSON_SERIALIZER); } export async function testMemoryLruPersistence( params: LruParams = LruParams.DEFAULT ): Promise { - return new MemoryPersistence(p => new MemoryLruDelegate(p, params)); + return new MemoryPersistence( + p => new MemoryLruDelegate(p, params), + JSON_SERIALIZER + ); } /** Clears the persistence in tests */ diff --git a/packages/firestore/test/unit/local/test_bundle_cache.ts b/packages/firestore/test/unit/local/test_bundle_cache.ts new file mode 100644 index 00000000000..e0ac098952c --- /dev/null +++ b/packages/firestore/test/unit/local/test_bundle_cache.ts @@ -0,0 +1,73 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Persistence } from '../../../src/local/persistence'; +import { BundleCache } from '../../../src/local/bundle_cache'; +import { Bundle, NamedQuery } from '../../../src/core/bundle'; +import * as bundleProto from '../../../src/protos/firestore_bundle_proto'; + +/** + * A wrapper around a BundleCache that automatically creates a + * transaction around every operation to reduce test boilerplate. + */ +export class TestBundleCache { + private readonly cache: BundleCache; + + constructor(private readonly persistence: Persistence) { + this.cache = persistence.getBundleCache(); + } + + getBundle(bundleId: string): Promise { + return this.persistence.runTransaction( + 'getBundle', + 'readonly', + transaction => { + return this.cache.getBundle(transaction, bundleId); + } + ); + } + + saveBundleMetadata(metadata: bundleProto.BundleMetadata): Promise { + return this.persistence.runTransaction( + 'saveBundleMetadata', + 'readwrite', + transaction => { + return this.cache.saveBundleMetadata(transaction, metadata); + } + ); + } + + getNamedQuery(name: string): Promise { + return this.persistence.runTransaction( + 'getNamedQuery', + 'readonly', + transaction => { + return this.cache.getNamedQuery(transaction, name); + } + ); + } + + setNamedQuery(query: bundleProto.NamedQuery): Promise { + return this.persistence.runTransaction( + 'setNamedQuery', + 'readwrite', + transaction => { + return this.cache.saveNamedQuery(transaction, query); + } + ); + } +} diff --git a/packages/firestore/test/unit/specs/spec_test_components.ts b/packages/firestore/test/unit/specs/spec_test_components.ts index 9fba3e611a4..1da4d480d8c 100644 --- a/packages/firestore/test/unit/specs/spec_test_components.ts +++ b/packages/firestore/test/unit/specs/spec_test_components.ts @@ -51,6 +51,9 @@ import { ViewSnapshot } from '../../../src/core/view_snapshot'; import { Query } from '../../../src/core/query'; import { Mutation } from '../../../src/model/mutation'; import { expect } from 'chai'; +import { + JSON_SERIALIZER +} from '../local/persistence_test_helpers'; /** * A test-only MemoryPersistence implementation that is able to inject @@ -161,7 +164,8 @@ export class MockMemoryComponentProvider extends MemoryComponentProvider { return new MockMemoryPersistence( this.gcEnabled ? MemoryEagerDelegate.factory - : p => new MemoryLruDelegate(p, LruParams.DEFAULT) + : p => new MemoryLruDelegate(p, LruParams.DEFAULT), + JSON_SERIALIZER ); } } From fff3d36dad514d493f0cd91c583ad61e77b51d76 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Mon, 1 Jun 2020 15:35:02 -0400 Subject: [PATCH 13/43] Add applyBundleDocuments to local store. --- packages/firestore/src/core/bundle.ts | 37 ++++ .../firestore/src/core/component_provider.ts | 9 +- .../local/indexeddb_remote_document_cache.ts | 8 +- packages/firestore/src/local/local_store.ts | 191 ++++++++++++------ .../src/local/memory_remote_document_cache.ts | 6 +- .../local/remote_document_change_buffer.ts | 47 ++--- .../test/unit/local/local_store.test.ts | 146 ++++++++++++- .../remote_document_change_buffer.test.ts | 9 - packages/firestore/test/util/helpers.ts | 35 ++++ 9 files changed, 382 insertions(+), 106 deletions(-) diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index 0571664b7f4..52bea47d813 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -17,6 +17,16 @@ import { Query } from './query'; import { SnapshotVersion } from './snapshot_version'; +import { JsonProtoSerializer } from '../remote/serializer'; +import { + firestoreV1ApiClientInterfaces, + Timestamp +} from '../protos/firestore_proto_api'; +import Document = firestoreV1ApiClientInterfaces.Document; +import { DocumentKey } from '../model/document_key'; +import { MaybeDocument, NoDocument } from '../model/document'; +import { BundledDocumentMetadata } from '../protos/firestore_bundle_proto'; +import { debugAssert } from '../util/assert'; /** * Represents a Firestore bundle saved by the SDK in its local storage. @@ -37,3 +47,30 @@ export interface NamedQuery { // When the results for this query are read to the saved bundle. readonly readTime: SnapshotVersion; } + +export class BundleConverter { + constructor(private serializer: JsonProtoSerializer) {} + + toDocumentKey(name: string): DocumentKey { + return this.serializer.fromName(name); + } + + toMaybeDocument( + metadata: BundledDocumentMetadata, + doc: Document | undefined + ): MaybeDocument { + if (metadata.exists) { + debugAssert(!!doc, 'Document is undefined when metadata.exist is true.'); + return this.serializer.fromDocument(doc!, false); + } else { + return new NoDocument( + this.toDocumentKey(metadata.name!), + this.toSnapshotVersion(metadata.readTime!) + ); + } + } + + toSnapshotVersion(time: Timestamp): SnapshotVersion { + return this.serializer.fromVersion(time); + } +} diff --git a/packages/firestore/src/core/component_provider.ts b/packages/firestore/src/core/component_provider.ts index 38ba5d2c763..94ec74d71ed 100644 --- a/packages/firestore/src/core/component_provider.ts +++ b/packages/firestore/src/core/component_provider.ts @@ -42,6 +42,7 @@ import { MemoryEagerDelegate, MemoryPersistence } from '../local/memory_persistence'; +import { BundleConverter } from './bundle'; const MEMORY_ONLY_PERSISTENCE_ERROR_MESSAGE = 'You are using the memory-only build of Firestore. Persistence support is ' + @@ -125,10 +126,12 @@ export class MemoryComponentProvider implements ComponentProvider { } createLocalStore(cfg: ComponentConfiguration): LocalStore { + const serializer = cfg.platform.newSerializer(cfg.databaseInfo.databaseId); return new LocalStore( this.persistence, new IndexFreeQueryEngine(), - cfg.initialUser + cfg.initialUser, + new BundleConverter(serializer) ); } @@ -209,10 +212,12 @@ export class IndexedDbComponentProvider extends MemoryComponentProvider { } createLocalStore(cfg: ComponentConfiguration): LocalStore { + const serializer = cfg.platform.newSerializer(cfg.databaseInfo.databaseId); return new MultiTabLocalStore( this.persistence, new IndexFreeQueryEngine(), - cfg.initialUser + cfg.initialUser, + new BundleConverter(serializer) ); } diff --git a/packages/firestore/src/local/indexeddb_remote_document_cache.ts b/packages/firestore/src/local/indexeddb_remote_document_cache.ts index cf073f2b766..ad423ed82a5 100644 --- a/packages/firestore/src/local/indexeddb_remote_document_cache.ts +++ b/packages/firestore/src/local/indexeddb_remote_document_cache.ts @@ -453,12 +453,12 @@ export class IndexedDbRemoteDocumentCache implements RemoteDocumentCache { ); if (maybeDocument) { debugAssert( - !this.readTime.isEqual(SnapshotVersion.min()), + !this.getReadTime(key).isEqual(SnapshotVersion.min()), 'Cannot add a document with a read time of zero' ); const doc = this.documentCache.serializer.toDbRemoteDocument( - maybeDocument, - this.readTime + maybeDocument.maybeDoc!, + this.getReadTime(key) ); collectionParents = collectionParents.add(key.path.popLast()); @@ -474,7 +474,7 @@ export class IndexedDbRemoteDocumentCache implements RemoteDocumentCache { // preserved in `getNewDocumentChanges()`. const deletedDoc = this.documentCache.serializer.toDbRemoteDocument( new NoDocument(key, SnapshotVersion.min()), - this.readTime + this.getReadTime(key) ); promises.push( this.documentCache.addEntry(transaction, key, deletedDoc) diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 361cbd48b7f..c5cc857167b 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -25,6 +25,8 @@ import { DocumentKeySet, documentKeySet, DocumentMap, + documentVersionMap, + DocumentVersionMap, maybeDocumentMap, MaybeDocumentMap } from '../model/collections'; @@ -67,6 +69,9 @@ import { IndexedDbRemoteDocumentCache } from './indexeddb_remote_document_cache' import { IndexedDbTargetCache } from './indexeddb_target_cache'; import { extractFieldMask } from '../model/object_value'; import { isIndexedDbTransactionError } from './simple_db'; +import { BundledDocumentMetadata } from '../protos/firestore_bundle_proto'; +import * as api from '../protos/firestore_proto_api'; +import { BundleConverter } from '../core/bundle'; const LOG_TAG = 'LocalStore'; @@ -195,7 +200,8 @@ export class LocalStore { /** Manages our in-memory or durable persistence. */ protected persistence: Persistence, private queryEngine: QueryEngine, - initialUser: User + initialUser: User, + private bundleConverter: BundleConverter ) { debugAssert( persistence.started, @@ -485,10 +491,6 @@ export class LocalStore { return this.persistence .runTransaction('Apply remote event', 'readwrite-primary', txn => { - const documentBuffer = this.remoteDocuments.newChangeBuffer({ - trackRemovals: true // Make sure document removals show up in `getNewDocumentChanges()` - }); - // Reset newTargetDataByTargetMap in case this transaction gets re-run. newTargetDataByTargetMap = this.targetDataByTarget; @@ -541,65 +543,31 @@ export class LocalStore { } }); + const documentBuffer = this.remoteDocuments.newChangeBuffer({ + trackRemovals: true // Make sure document removals show up in `getNewDocumentChanges()` + }); let changedDocs = maybeDocumentMap(); let updatedKeys = documentKeySet(); remoteEvent.documentUpdates.forEach((key, doc) => { updatedKeys = updatedKeys.add(key); + if (remoteEvent.resolvedLimboDocuments.has(key)) { + promises.push( + this.persistence.referenceDelegate.updateLimboDocument(txn, key) + ); + } }); // Each loop iteration only affects its "own" doc, so it's safe to get all the remote // documents in advance in a single call. promises.push( - documentBuffer.getEntries(txn, updatedKeys).next(existingDocs => { - remoteEvent.documentUpdates.forEach((key, doc) => { - const existingDoc = existingDocs.get(key); - - // Note: The order of the steps below is important, since we want - // to ensure that rejected limbo resolutions (which fabricate - // NoDocuments with SnapshotVersion.min()) never add documents to - // cache. - if ( - doc instanceof NoDocument && - doc.version.isEqual(SnapshotVersion.min()) - ) { - // NoDocuments with SnapshotVersion.min() are used in manufactured - // events. We remove these documents from cache since we lost - // access. - documentBuffer.removeEntry(key, remoteVersion); - changedDocs = changedDocs.insert(key, doc); - } else if ( - existingDoc == null || - doc.version.compareTo(existingDoc.version) > 0 || - (doc.version.compareTo(existingDoc.version) === 0 && - existingDoc.hasPendingWrites) - ) { - debugAssert( - !SnapshotVersion.min().isEqual(remoteVersion), - 'Cannot add a document when the remote version is zero' - ); - documentBuffer.addEntry(doc, remoteVersion); - changedDocs = changedDocs.insert(key, doc); - } else { - logDebug( - LOG_TAG, - 'Ignoring outdated watch update for ', - key, - '. Current version:', - existingDoc.version, - ' Watch version:', - doc.version - ); - } - - if (remoteEvent.resolvedLimboDocuments.has(key)) { - promises.push( - this.persistence.referenceDelegate.updateLimboDocument( - txn, - key - ) - ); - } - }); + this.applyDocuments( + documentBuffer, + txn, + updatedKeys, + remoteEvent.documentUpdates, + remoteVersion + ).next(result => { + changedDocs = result; }) ); @@ -642,6 +610,114 @@ export class LocalStore { }); } + applyBundledDocuments( + documents: Array<[BundledDocumentMetadata, api.Document | undefined]> + ): Promise { + let updatedKeys = documentKeySet(); + let documentMap = maybeDocumentMap(); + let versionMap = documentVersionMap(); + for (const [metadata, doc] of documents) { + const documentKey = this.bundleConverter.toDocumentKey(metadata.name!); + updatedKeys = updatedKeys.add(documentKey); + documentMap = documentMap.insert( + documentKey, + this.bundleConverter.toMaybeDocument(metadata, doc) + ); + versionMap = versionMap.insert( + documentKey, + this.bundleConverter.toSnapshotVersion(metadata.readTime!) + ); + } + + const documentBuffer = this.remoteDocuments.newChangeBuffer({ + trackRemovals: true // Make sure document removals show up in `getNewDocumentChanges()` + }); + return this.persistence.runTransaction( + 'Apply bundle documents', + 'readwrite-primary', + txn => { + return this.applyDocuments( + documentBuffer, + txn, + updatedKeys, + documentMap, + versionMap + ) + .next(changedDocs => { + documentBuffer.apply(txn); + return changedDocs; + }) + .next(changedDocs => { + return this.localDocuments.getLocalViewOfDocuments( + txn, + changedDocs + ); + }); + } + ); + } + + private applyDocuments( + documentBuffer: RemoteDocumentChangeBuffer, + txn: PersistenceTransaction, + updatedKeys: DocumentKeySet, + documents: MaybeDocumentMap, + remoteVersion: SnapshotVersion | DocumentVersionMap + ): PersistencePromise { + const universalReadTime = remoteVersion instanceof SnapshotVersion; + return documentBuffer.getEntries(txn, updatedKeys).next(existingDocs => { + let changedDocs = maybeDocumentMap(); + documents.forEach((key, doc) => { + const existingDoc = existingDocs.get(key); + let docReadVersion: SnapshotVersion | null = null; + if (universalReadTime) { + docReadVersion = remoteVersion as SnapshotVersion; + } else { + docReadVersion = (remoteVersion as DocumentVersionMap).get(key); + } + debugAssert(!!docReadVersion, 'Document read version must exist'); + + // Note: The order of the steps below is important, since we want + // to ensure that rejected limbo resolutions (which fabricate + // NoDocuments with SnapshotVersion.min()) never add documents to + // cache. + if ( + doc instanceof NoDocument && + doc.version.isEqual(SnapshotVersion.min()) + ) { + // NoDocuments with SnapshotVersion.min() are used in manufactured + // events. We remove these documents from cache since we lost + // access. + documentBuffer.removeEntry(key, docReadVersion!); + changedDocs = changedDocs.insert(key, doc); + } else if ( + existingDoc == null || + doc.version.compareTo(existingDoc.version) > 0 || + (doc.version.compareTo(existingDoc.version) === 0 && + existingDoc.hasPendingWrites) + ) { + debugAssert( + !SnapshotVersion.min().isEqual(docReadVersion!), + 'Cannot add a document when the remote version is zero' + ); + documentBuffer.addEntry(doc, docReadVersion!); + changedDocs = changedDocs.insert(key, doc); + } else { + logDebug( + LOG_TAG, + 'Ignoring outdated watch update for ', + key, + '. Current version:', + existingDoc.version, + ' Watch version:', + doc.version + ); + } + }); + return changedDocs; + }); + } + /** * Returns true if the newTargetData should be persisted during an update of * an active target. TargetData should always be persisted when a target is @@ -1008,9 +1084,10 @@ export class MultiTabLocalStore extends LocalStore { constructor( protected persistence: IndexedDbPersistence, queryEngine: QueryEngine, - initialUser: User + initialUser: User, + bundleConverter: BundleConverter ) { - super(persistence, queryEngine, initialUser); + super(persistence, queryEngine, initialUser, bundleConverter); this.mutationQueue = persistence.getMutationQueue(initialUser); this.remoteDocuments = persistence.getRemoteDocumentCache(); diff --git a/packages/firestore/src/local/memory_remote_document_cache.ts b/packages/firestore/src/local/memory_remote_document_cache.ts index c8b147e6785..3d08abe2f9d 100644 --- a/packages/firestore/src/local/memory_remote_document_cache.ts +++ b/packages/firestore/src/local/memory_remote_document_cache.ts @@ -202,7 +202,11 @@ export class MemoryRemoteDocumentCache implements RemoteDocumentCache { this.changes.forEach((key, doc) => { if (doc) { promises.push( - this.documentCache.addEntry(transaction, doc, this.readTime) + this.documentCache.addEntry( + transaction, + doc.maybeDoc!, + doc.readTime! + ) ); } else { this.documentCache.removeEntry(key); diff --git a/packages/firestore/src/local/remote_document_change_buffer.ts b/packages/firestore/src/local/remote_document_change_buffer.ts index b434900afdb..c9fe583a144 100644 --- a/packages/firestore/src/local/remote_document_change_buffer.ts +++ b/packages/firestore/src/local/remote_document_change_buffer.ts @@ -25,6 +25,13 @@ import { PersistenceTransaction } from './persistence'; import { PersistencePromise } from './persistence_promise'; import { SnapshotVersion } from '../core/snapshot_version'; +class RemoteDocumentChange { + constructor( + readonly maybeDoc: MaybeDocument | null, + readonly readTime: SnapshotVersion + ) {} +} + /** * An in-memory buffer of entries to be written to a RemoteDocumentCache. * It can be used to batch up a set of changes to be written to the cache, but @@ -44,12 +51,9 @@ export abstract class RemoteDocumentChangeBuffer { // existing cache entry should be removed). protected changes: ObjectMap< DocumentKey, - MaybeDocument | null + RemoteDocumentChange | null > = new ObjectMap(key => key.toString()); - // The read time to use for all added documents in this change buffer. - private _readTime: SnapshotVersion | undefined; - private changesApplied = false; protected abstract getFromCache( @@ -66,23 +70,11 @@ export abstract class RemoteDocumentChangeBuffer { transaction: PersistenceTransaction ): PersistencePromise; - protected set readTime(value: SnapshotVersion) { - // Right now (for simplicity) we just track a single readTime for all the - // added entries since we expect them to all be the same, but we could - // rework to store per-entry readTimes if necessary. - debugAssert( - this._readTime === undefined || this._readTime.isEqual(value), - 'All changes in a RemoteDocumentChangeBuffer must have the same read time' - ); - this._readTime = value; - } - - protected get readTime(): SnapshotVersion { - debugAssert( - this._readTime !== undefined, - 'Read time is not set. All removeEntry() calls must include a readTime if `trackRemovals` is used.' - ); - return this._readTime; + protected getReadTime(key: DocumentKey): SnapshotVersion { + if (this.changes.get(key)) { + return this.changes.get(key)!.readTime; + } + return SnapshotVersion.min(); } /** @@ -93,8 +85,10 @@ export abstract class RemoteDocumentChangeBuffer { */ addEntry(maybeDocument: MaybeDocument, readTime: SnapshotVersion): void { this.assertNotApplied(); - this.readTime = readTime; - this.changes.set(maybeDocument.key, maybeDocument); + this.changes.set( + maybeDocument.key, + new RemoteDocumentChange(maybeDocument, readTime) + ); } /** @@ -105,9 +99,6 @@ export abstract class RemoteDocumentChangeBuffer { */ removeEntry(key: DocumentKey, readTime?: SnapshotVersion): void { this.assertNotApplied(); - if (readTime) { - this.readTime = readTime; - } this.changes.set(key, null); } @@ -129,7 +120,9 @@ export abstract class RemoteDocumentChangeBuffer { this.assertNotApplied(); const bufferedEntry = this.changes.get(documentKey); if (bufferedEntry !== undefined) { - return PersistencePromise.resolve(bufferedEntry); + return PersistencePromise.resolve( + (bufferedEntry || { maybeDoc: null }).maybeDoc + ); } else { return this.getFromCache(transaction, documentKey); } diff --git a/packages/firestore/test/unit/local/local_store.test.ts b/packages/firestore/test/unit/local/local_store.test.ts index c7a3ee3b6a6..fc2737a8edc 100644 --- a/packages/firestore/test/unit/local/local_store.test.ts +++ b/packages/firestore/test/unit/local/local_store.test.ts @@ -74,6 +74,8 @@ import { patchMutation, path, setMutation, + bundledDocuments, + TestBundledDocuments, TestSnapshotVersion, transformMutation, unknownDoc, @@ -83,6 +85,11 @@ import { import { CountingQueryEngine, QueryEngineType } from './counting_query_engine'; import * as persistenceHelpers from './persistence_test_helpers'; import { ByteString } from '../../../src/util/byte_string'; +import { BundleConverter } from '../../../src/core/bundle'; +import { JSON_SERIALIZER } from './persistence_test_helpers'; +import { BundledDocumentMetadata } from '../../../src/protos/firestore_bundle_proto'; +import { firestoreV1ApiClientInterfaces } from '../../../src/protos/firestore_proto_api'; +import Document = firestoreV1ApiClientInterfaces.Document; export interface LocalStoreComponents { queryEngine: CountingQueryEngine; @@ -111,7 +118,12 @@ class LocalStoreTester { } after( - op: Mutation | Mutation[] | RemoteEvent | LocalViewChanges + op: + | Mutation + | Mutation[] + | RemoteEvent + | LocalViewChanges + | TestBundledDocuments ): LocalStoreTester { if (op instanceof Mutation) { return this.afterMutations([op]); @@ -119,8 +131,10 @@ class LocalStoreTester { return this.afterMutations(op); } else if (op instanceof LocalViewChanges) { return this.afterViewChanges(op); - } else { + } else if (op instanceof RemoteEvent) { return this.afterRemoteEvent(op); + } else { + return this.afterBundleDocuments(op.documents); } } @@ -153,6 +167,21 @@ class LocalStoreTester { return this; } + afterBundleDocuments( + documents: Array<[BundledDocumentMetadata, Document | undefined]> + ): LocalStoreTester { + this.prepareNextStep(); + + this.promiseChain = this.promiseChain + .then(() => { + return this.localStore.applyBundledDocuments(documents); + }) + .then((result: MaybeDocumentMap) => { + this.lastChanges = result; + }); + return this; + } + afterViewChanges(viewChanges: LocalViewChanges): LocalStoreTester { this.prepareNextStep(); @@ -404,7 +433,8 @@ describe('LocalStore w/ Memory Persistence (SimpleQueryEngine)', () => { const localStore = new LocalStore( persistence, queryEngine, - User.UNAUTHENTICATED + User.UNAUTHENTICATED, + new BundleConverter(JSON_SERIALIZER) ); return { queryEngine, persistence, localStore }; } @@ -422,7 +452,8 @@ describe('LocalStore w/ Memory Persistence (IndexFreeQueryEngine)', () => { const localStore = new LocalStore( persistence, queryEngine, - User.UNAUTHENTICATED + User.UNAUTHENTICATED, + new BundleConverter(JSON_SERIALIZER) ); return { queryEngine, persistence, localStore }; } @@ -448,7 +479,8 @@ describe('LocalStore w/ IndexedDB Persistence (SimpleQueryEngine)', () => { const localStore = new MultiTabLocalStore( persistence, queryEngine, - User.UNAUTHENTICATED + User.UNAUTHENTICATED, + new BundleConverter(JSON_SERIALIZER) ); await localStore.start(); return { queryEngine, persistence, localStore }; @@ -475,7 +507,8 @@ describe('LocalStore w/ IndexedDB Persistence (IndexFreeQueryEngine)', () => { const localStore = new MultiTabLocalStore( persistence, queryEngine, - User.UNAUTHENTICATED + User.UNAUTHENTICATED, + new BundleConverter(JSON_SERIALIZER) ); await localStore.start(); return { queryEngine, persistence, localStore }; @@ -1486,6 +1519,107 @@ function genericLocalStoreTests( .finish(); }); + it('handles saving bundled documents', () => { + return expectLocalStore() + .after( + bundledDocuments([ + doc('foo/bar', 1, { sum: 1337 }), + deletedDoc('foo/bar1', 1) + ]) + ) + .toReturnChanged( + doc('foo/bar', 1, { sum: 1337 }), + deletedDoc('foo/bar1', 1) + ) + .toContain(doc('foo/bar', 1, { sum: 1337 })) + .toContain(deletedDoc('foo/bar1', 1)) + .finish(); + }); + + it('handles saving bundled documents with newer existing version', () => { + const query = Query.atPath(path('foo')); + return expectLocalStore() + .afterAllocatingQuery(query) + .toReturnTargetId(2) + .after(docAddedRemoteEvent(doc('foo/bar', 2, { sum: 1337 }), [2])) + .toContain(doc('foo/bar', 2, { sum: 1337 })) + .after( + bundledDocuments([ + doc('foo/bar', 1, { sum: 1336 }), + deletedDoc('foo/bar1', 1) + ]) + ) + .toReturnChanged(deletedDoc('foo/bar1', 1)) + .toContain(doc('foo/bar', 2, { sum: 1337 })) + .toContain(deletedDoc('foo/bar1', 1)) + .finish(); + }); + + it('handles saving bundled documents with older existing version', () => { + const query = Query.atPath(path('foo')); + return expectLocalStore() + .afterAllocatingQuery(query) + .toReturnTargetId(2) + .after(docAddedRemoteEvent(doc('foo/bar1', 1, { val: 'to-delete' }), [2])) + .toContain(doc('foo/bar1', 1, { val: 'to-delete' })) + .after( + bundledDocuments([ + doc('foo/bar', 1, { sum: 1336 }), + deletedDoc('foo/bar1', 2) + ]) + ) + .toReturnChanged( + doc('foo/bar', 1, { sum: 1336 }), + deletedDoc('foo/bar1', 2) + ) + .toContain(doc('foo/bar', 1, { sum: 1336 })) + .toContain(deletedDoc('foo/bar1', 2)) + .finish(); + }); + + it('handles MergeMutation with Transform -> BundledDocuments', () => { + const query = Query.atPath(path('foo')); + return expectLocalStore() + .afterAllocatingQuery(query) + .toReturnTargetId(2) + .afterMutations([ + patchMutation('foo/bar', {}, Precondition.none()), + transformMutation('foo/bar', { sum: FieldValue.increment(1) }) + ]) + .toReturnChanged( + doc('foo/bar', 0, { sum: 1 }, { hasLocalMutations: true }) + ) + .toContain(doc('foo/bar', 0, { sum: 1 }, { hasLocalMutations: true })) + .after(bundledDocuments([doc('foo/bar', 1, { sum: 1337 })])) + .toReturnChanged( + doc('foo/bar', 1, { sum: 1 }, { hasLocalMutations: true }) + ) + .toContain(doc('foo/bar', 1, { sum: 1 }, { hasLocalMutations: true })) + .finish(); + }); + + it('handles PatchMutation with Transform -> BundledDocuments', () => { + // Note: This test reflects the current behavior, but it may be preferable + // to replay the mutation once we receive the first value from the backend. + + const query = Query.atPath(path('foo')); + return expectLocalStore() + .afterAllocatingQuery(query) + .toReturnTargetId(2) + .afterMutations([ + patchMutation('foo/bar', {}), + transformMutation('foo/bar', { sum: FieldValue.increment(1) }) + ]) + .toReturnChanged(deletedDoc('foo/bar', 0)) + .toNotContain('foo/bar') + .after(bundledDocuments([doc('foo/bar', 1, { sum: 1337 })])) + .toReturnChanged( + doc('foo/bar', 1, { sum: 1 }, { hasLocalMutations: true }) + ) + .toContain(doc('foo/bar', 1, { sum: 1 }, { hasLocalMutations: true })) + .finish(); + }); + it('computes highest unacknowledged batch id correctly', () => { return expectLocalStore() .toReturnHighestUnacknowledgeBatchId(BATCHID_UNKNOWN) diff --git a/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts b/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts index ec77f35695d..043e5905434 100644 --- a/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts +++ b/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts @@ -116,13 +116,4 @@ describe('RemoteDocumentChangeBuffer', () => { .catch(() => errors++) .then(() => expect(errors).to.equal(2)); }); - - it('cannot add documents with different read times', async () => { - // This test merely validates an assert that was added to the - // RemoteDocumentChangeBuffer to simplify our tracking of document - // read times. If we do need to track documents with different read - // times, this test should simply be removed. - buffer.addEntry(INITIAL_DOC, version(1)); - expect(() => buffer.addEntry(INITIAL_DOC, version(2))).to.throw(); - }); }); diff --git a/packages/firestore/test/util/helpers.ts b/packages/firestore/test/util/helpers.ts index 8dc7545daf2..02c7f6ab3ba 100644 --- a/packages/firestore/test/util/helpers.ts +++ b/packages/firestore/test/util/helpers.ts @@ -92,6 +92,8 @@ import { Timestamp } from '../../src/api/timestamp'; import { DocumentReference, Firestore } from '../../src/api/database'; import { DeleteFieldValueImpl } from '../../src/api/field_value'; import { Code, FirestoreError } from '../../src/util/error'; +import { BundledDocumentMetadata } from '../../src/protos/firestore_bundle_proto'; +import { JSON_SERIALIZER } from '../unit/local/persistence_test_helpers'; /* eslint-disable no-restricted-globals */ @@ -401,6 +403,39 @@ export function docUpdateRemoteEvent( return aggregator.createRemoteEvent(doc.version); } +export class TestBundledDocuments { + constructor( + public documents: Array<[BundledDocumentMetadata, api.Document | undefined]> + ) {} +} + +export function bundledDocuments(documents: MaybeDocument[]) { + let result: Array<[BundledDocumentMetadata, api.Document | undefined]> = []; + for (const d of documents) { + if (d instanceof NoDocument) { + result.push([ + { + name: JSON_SERIALIZER.toName(d.key), + readTime: JSON_SERIALIZER.toVersion(d.version), + exists: false + }, + undefined + ]); + } else if (d instanceof Document) { + result.push([ + { + name: JSON_SERIALIZER.toName(d.key), + readTime: JSON_SERIALIZER.toVersion(d.version), + exists: true + }, + JSON_SERIALIZER.toDocument(d) + ]); + } + } + + return new TestBundledDocuments(result); +} + export function updateMapping( snapshotVersion: SnapshotVersion, added: Array, From fb762de46cdc6b2af65a81f168594ddefc517ddd Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Mon, 1 Jun 2020 16:50:41 -0400 Subject: [PATCH 14/43] Add rest of bundle service to localstore Fix change buffer bug --- packages/firestore/src/core/bundle.ts | 18 +++--- .../local/indexeddb_remote_document_cache.ts | 6 +- packages/firestore/src/local/local_store.ts | 56 ++++++++++++++++++- .../src/local/memory_remote_document_cache.ts | 4 +- .../local/remote_document_change_buffer.ts | 10 ++-- .../test/unit/local/local_store.test.ts | 6 +- packages/firestore/test/util/helpers.ts | 11 ++-- 7 files changed, 80 insertions(+), 31 deletions(-) diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index 52bea47d813..bcd8bedcfe0 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -18,14 +18,10 @@ import { Query } from './query'; import { SnapshotVersion } from './snapshot_version'; import { JsonProtoSerializer } from '../remote/serializer'; -import { - firestoreV1ApiClientInterfaces, - Timestamp -} from '../protos/firestore_proto_api'; -import Document = firestoreV1ApiClientInterfaces.Document; +import * as bundleProto from '../protos/firestore_bundle_proto'; +import * as api from '../protos/firestore_proto_api'; import { DocumentKey } from '../model/document_key'; import { MaybeDocument, NoDocument } from '../model/document'; -import { BundledDocumentMetadata } from '../protos/firestore_bundle_proto'; import { debugAssert } from '../util/assert'; /** @@ -48,6 +44,10 @@ export interface NamedQuery { readonly readTime: SnapshotVersion; } +export type BundledDocuments = Array< + [bundleProto.BundledDocumentMetadata, api.Document | undefined] +>; + export class BundleConverter { constructor(private serializer: JsonProtoSerializer) {} @@ -56,8 +56,8 @@ export class BundleConverter { } toMaybeDocument( - metadata: BundledDocumentMetadata, - doc: Document | undefined + metadata: bundleProto.BundledDocumentMetadata, + doc: api.Document | undefined ): MaybeDocument { if (metadata.exists) { debugAssert(!!doc, 'Document is undefined when metadata.exist is true.'); @@ -70,7 +70,7 @@ export class BundleConverter { } } - toSnapshotVersion(time: Timestamp): SnapshotVersion { + toSnapshotVersion(time: api.Timestamp): SnapshotVersion { return this.serializer.fromVersion(time); } } diff --git a/packages/firestore/src/local/indexeddb_remote_document_cache.ts b/packages/firestore/src/local/indexeddb_remote_document_cache.ts index ad423ed82a5..8b78428fae0 100644 --- a/packages/firestore/src/local/indexeddb_remote_document_cache.ts +++ b/packages/firestore/src/local/indexeddb_remote_document_cache.ts @@ -445,19 +445,19 @@ export class IndexedDbRemoteDocumentCache implements RemoteDocumentCache { primitiveComparator(l.canonicalString(), r.canonicalString()) ); - this.changes.forEach((key, maybeDocument) => { + this.changes.forEach((key, documentChange) => { const previousSize = this.documentSizes.get(key); debugAssert( previousSize !== undefined, `Cannot modify a document that wasn't read (for ${key})` ); - if (maybeDocument) { + if (documentChange.maybeDoc) { debugAssert( !this.getReadTime(key).isEqual(SnapshotVersion.min()), 'Cannot add a document with a read time of zero' ); const doc = this.documentCache.serializer.toDbRemoteDocument( - maybeDocument.maybeDoc!, + documentChange.maybeDoc!, this.getReadTime(key) ); collectionParents = collectionParents.add(key.path.popLast()); diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index c5cc857167b..4d53f9171c8 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -69,9 +69,15 @@ import { IndexedDbRemoteDocumentCache } from './indexeddb_remote_document_cache' import { IndexedDbTargetCache } from './indexeddb_target_cache'; import { extractFieldMask } from '../model/object_value'; import { isIndexedDbTransactionError } from './simple_db'; -import { BundledDocumentMetadata } from '../protos/firestore_bundle_proto'; +import * as bundleProto from '../protos/firestore_bundle_proto'; import * as api from '../protos/firestore_proto_api'; -import { BundleConverter } from '../core/bundle'; +import { + Bundle, + BundleConverter, + BundledDocuments, + NamedQuery +} from '../core/bundle'; +import { BundleCache } from './bundle_cache'; const LOG_TAG = 'LocalStore'; @@ -173,6 +179,9 @@ export class LocalStore { /** Maps a target to its `TargetData`. */ protected targetCache: TargetCache; + /** The set of all cached bundle metadata and named queries. */ + protected bundleCache: BundleCache; + /** * Maps a targetID to data about its target. * @@ -210,6 +219,7 @@ export class LocalStore { this.mutationQueue = persistence.getMutationQueue(initialUser); this.remoteDocuments = persistence.getRemoteDocumentCache(); this.targetCache = persistence.getTargetCache(); + this.bundleCache = persistence.getBundleCache(); this.localDocuments = new LocalDocumentsView( this.remoteDocuments, this.mutationQueue, @@ -611,7 +621,7 @@ export class LocalStore { } applyBundledDocuments( - documents: Array<[BundledDocumentMetadata, api.Document | undefined]> + documents: BundledDocuments ): Promise { let updatedKeys = documentKeySet(); let documentMap = maybeDocumentMap(); @@ -718,6 +728,46 @@ export class LocalStore { }); } + getBundle(bundleId: string): Promise { + return this.persistence.runTransaction( + 'Get bundle', + 'readonly', + transaction => { + return this.bundleCache.getBundle(transaction, bundleId); + } + ); + } + + saveBundle(bundleMetadata: bundleProto.BundleMetadata): Promise { + return this.persistence.runTransaction( + 'Save bundle', + 'readwrite', + transaction => { + return this.bundleCache.saveBundleMetadata(transaction, bundleMetadata); + } + ); + } + + getNamedQuery(queryName: string): Promise { + return this.persistence.runTransaction( + 'Get named query', + 'readonly', + transaction => { + return this.bundleCache.getNamedQuery(transaction, queryName); + } + ); + } + + saveNamedQuery(query: bundleProto.NamedQuery): Promise { + return this.persistence.runTransaction( + 'Save named query', + 'readwrite', + transaction => { + return this.bundleCache.saveNamedQuery(transaction, query); + } + ); + } + /** * Returns true if the newTargetData should be persisted during an update of * an active target. TargetData should always be persisted when a target is diff --git a/packages/firestore/src/local/memory_remote_document_cache.ts b/packages/firestore/src/local/memory_remote_document_cache.ts index 3d08abe2f9d..ced4b868774 100644 --- a/packages/firestore/src/local/memory_remote_document_cache.ts +++ b/packages/firestore/src/local/memory_remote_document_cache.ts @@ -200,12 +200,12 @@ export class MemoryRemoteDocumentCache implements RemoteDocumentCache { ): PersistencePromise { const promises: Array> = []; this.changes.forEach((key, doc) => { - if (doc) { + if (doc && doc.maybeDoc) { promises.push( this.documentCache.addEntry( transaction, doc.maybeDoc!, - doc.readTime! + this.getReadTime(key) ) ); } else { diff --git a/packages/firestore/src/local/remote_document_change_buffer.ts b/packages/firestore/src/local/remote_document_change_buffer.ts index c9fe583a144..9aabb4af2b7 100644 --- a/packages/firestore/src/local/remote_document_change_buffer.ts +++ b/packages/firestore/src/local/remote_document_change_buffer.ts @@ -28,7 +28,7 @@ import { SnapshotVersion } from '../core/snapshot_version'; class RemoteDocumentChange { constructor( readonly maybeDoc: MaybeDocument | null, - readonly readTime: SnapshotVersion + readonly readTime?: SnapshotVersion ) {} } @@ -51,7 +51,7 @@ export abstract class RemoteDocumentChangeBuffer { // existing cache entry should be removed). protected changes: ObjectMap< DocumentKey, - RemoteDocumentChange | null + RemoteDocumentChange > = new ObjectMap(key => key.toString()); private changesApplied = false; @@ -71,8 +71,8 @@ export abstract class RemoteDocumentChangeBuffer { ): PersistencePromise; protected getReadTime(key: DocumentKey): SnapshotVersion { - if (this.changes.get(key)) { - return this.changes.get(key)!.readTime; + if (this.changes.get(key) && this.changes.get(key)!.readTime) { + return this.changes.get(key)!.readTime!; } return SnapshotVersion.min(); } @@ -99,7 +99,7 @@ export abstract class RemoteDocumentChangeBuffer { */ removeEntry(key: DocumentKey, readTime?: SnapshotVersion): void { this.assertNotApplied(); - this.changes.set(key, null); + this.changes.set(key, new RemoteDocumentChange(null, readTime)); } /** diff --git a/packages/firestore/test/unit/local/local_store.test.ts b/packages/firestore/test/unit/local/local_store.test.ts index fc2737a8edc..1967262a236 100644 --- a/packages/firestore/test/unit/local/local_store.test.ts +++ b/packages/firestore/test/unit/local/local_store.test.ts @@ -85,7 +85,7 @@ import { import { CountingQueryEngine, QueryEngineType } from './counting_query_engine'; import * as persistenceHelpers from './persistence_test_helpers'; import { ByteString } from '../../../src/util/byte_string'; -import { BundleConverter } from '../../../src/core/bundle'; +import { BundleConverter, BundledDocuments } from '../../../src/core/bundle'; import { JSON_SERIALIZER } from './persistence_test_helpers'; import { BundledDocumentMetadata } from '../../../src/protos/firestore_bundle_proto'; import { firestoreV1ApiClientInterfaces } from '../../../src/protos/firestore_proto_api'; @@ -167,9 +167,7 @@ class LocalStoreTester { return this; } - afterBundleDocuments( - documents: Array<[BundledDocumentMetadata, Document | undefined]> - ): LocalStoreTester { + afterBundleDocuments(documents: BundledDocuments): LocalStoreTester { this.prepareNextStep(); this.promiseChain = this.promiseChain diff --git a/packages/firestore/test/util/helpers.ts b/packages/firestore/test/util/helpers.ts index 02c7f6ab3ba..d903944347d 100644 --- a/packages/firestore/test/util/helpers.ts +++ b/packages/firestore/test/util/helpers.ts @@ -94,6 +94,7 @@ import { DeleteFieldValueImpl } from '../../src/api/field_value'; import { Code, FirestoreError } from '../../src/util/error'; import { BundledDocumentMetadata } from '../../src/protos/firestore_bundle_proto'; import { JSON_SERIALIZER } from '../unit/local/persistence_test_helpers'; +import { BundledDocuments } from '../../src/core/bundle'; /* eslint-disable no-restricted-globals */ @@ -404,13 +405,13 @@ export function docUpdateRemoteEvent( } export class TestBundledDocuments { - constructor( - public documents: Array<[BundledDocumentMetadata, api.Document | undefined]> - ) {} + constructor(public documents: BundledDocuments) {} } -export function bundledDocuments(documents: MaybeDocument[]) { - let result: Array<[BundledDocumentMetadata, api.Document | undefined]> = []; +export function bundledDocuments( + documents: MaybeDocument[] +): TestBundledDocuments { + let result: BundledDocuments = []; for (const d of documents) { if (d instanceof NoDocument) { result.push([ From 1ec4182732ea261c447addc4ae18d8a94a882954 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 2 Jun 2020 11:30:08 -0400 Subject: [PATCH 15/43] Simplify change buffer get read time logic. --- .../src/local/remote_document_change_buffer.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/firestore/src/local/remote_document_change_buffer.ts b/packages/firestore/src/local/remote_document_change_buffer.ts index 9aabb4af2b7..747cd35bff6 100644 --- a/packages/firestore/src/local/remote_document_change_buffer.ts +++ b/packages/firestore/src/local/remote_document_change_buffer.ts @@ -71,8 +71,13 @@ export abstract class RemoteDocumentChangeBuffer { ): PersistencePromise; protected getReadTime(key: DocumentKey): SnapshotVersion { - if (this.changes.get(key) && this.changes.get(key)!.readTime) { - return this.changes.get(key)!.readTime!; + const change = this.changes.get(key); + if (change) { + debugAssert( + !!change.readTime, + `Read time is not set for ${key}. All removeEntry() calls must include a readTime if 'trackRemovals' is used.` + ); + return change.readTime!; } return SnapshotVersion.min(); } @@ -121,7 +126,7 @@ export abstract class RemoteDocumentChangeBuffer { const bufferedEntry = this.changes.get(documentKey); if (bufferedEntry !== undefined) { return PersistencePromise.resolve( - (bufferedEntry || { maybeDoc: null }).maybeDoc + bufferedEntry.maybeDoc ); } else { return this.getFromCache(transaction, documentKey); From cd3ab7abee88c45ed27555f5175d3ca61176bf0a Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 2 Jun 2020 11:51:23 -0400 Subject: [PATCH 16/43] Fix lint errors --- packages/firestore/src/local/local_store.ts | 1 - packages/firestore/test/unit/local/local_store.test.ts | 3 --- packages/firestore/test/util/helpers.ts | 3 +-- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 4d53f9171c8..69bef566fec 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -70,7 +70,6 @@ import { IndexedDbTargetCache } from './indexeddb_target_cache'; import { extractFieldMask } from '../model/object_value'; import { isIndexedDbTransactionError } from './simple_db'; import * as bundleProto from '../protos/firestore_bundle_proto'; -import * as api from '../protos/firestore_proto_api'; import { Bundle, BundleConverter, diff --git a/packages/firestore/test/unit/local/local_store.test.ts b/packages/firestore/test/unit/local/local_store.test.ts index 1967262a236..6b197525897 100644 --- a/packages/firestore/test/unit/local/local_store.test.ts +++ b/packages/firestore/test/unit/local/local_store.test.ts @@ -87,9 +87,6 @@ import * as persistenceHelpers from './persistence_test_helpers'; import { ByteString } from '../../../src/util/byte_string'; import { BundleConverter, BundledDocuments } from '../../../src/core/bundle'; import { JSON_SERIALIZER } from './persistence_test_helpers'; -import { BundledDocumentMetadata } from '../../../src/protos/firestore_bundle_proto'; -import { firestoreV1ApiClientInterfaces } from '../../../src/protos/firestore_proto_api'; -import Document = firestoreV1ApiClientInterfaces.Document; export interface LocalStoreComponents { queryEngine: CountingQueryEngine; diff --git a/packages/firestore/test/util/helpers.ts b/packages/firestore/test/util/helpers.ts index d903944347d..58d0620e647 100644 --- a/packages/firestore/test/util/helpers.ts +++ b/packages/firestore/test/util/helpers.ts @@ -92,7 +92,6 @@ import { Timestamp } from '../../src/api/timestamp'; import { DocumentReference, Firestore } from '../../src/api/database'; import { DeleteFieldValueImpl } from '../../src/api/field_value'; import { Code, FirestoreError } from '../../src/util/error'; -import { BundledDocumentMetadata } from '../../src/protos/firestore_bundle_proto'; import { JSON_SERIALIZER } from '../unit/local/persistence_test_helpers'; import { BundledDocuments } from '../../src/core/bundle'; @@ -411,7 +410,7 @@ export class TestBundledDocuments { export function bundledDocuments( documents: MaybeDocument[] ): TestBundledDocuments { - let result: BundledDocuments = []; + const result: BundledDocuments = []; for (const d of documents) { if (d instanceof NoDocument) { result.push([ From d991c7545e9b79d39cfbde7b8e314acddadfceae Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 2 Jun 2020 13:41:31 -0400 Subject: [PATCH 17/43] Add comments. --- packages/firestore/src/local/local_store.ts | 39 +++++++++++++++++-- .../local/remote_document_change_buffer.ts | 5 +++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 69bef566fec..4d843e03bf3 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -487,9 +487,10 @@ export class LocalStore { } /** - * Update the "ground-state" (remote) documents. We assume that the remote - * event reflects any write batches that have been acknowledged or rejected - * (i.e. we do not re-apply local mutations to updates from this event). + * Applies the documents from a remote event to the "ground-state" (remote) + * documents. We assume that the remote event reflects any write batches that + * have been acknowledged or rejected (i.e. we do not re-apply local + * mutations to updates from this event). * * LocalDocuments are re-calculated if there are remaining mutations in the * queue. @@ -619,6 +620,13 @@ export class LocalStore { }); } + /** + * Applies the documents from a bundle to the "ground-state" (remote) + * documents. + * + * LocalDocuments are re-calculated if there are remaining mutations in the + * queue. + */ applyBundledDocuments( documents: BundledDocuments ): Promise { @@ -666,6 +674,16 @@ export class LocalStore { ); } + /** + * Applies documents to remote document cache, returns the document changes + * resulting from applying those documents. + * + * @param updatedKeys Keys of the documents to be applied. + * @param documents Documents to be applied. + * @param remoteVersion The read time of the documents to be applied, it is + * a `SnapshotVersion` if all documents have the same read time, or a `DocumentVersionMap` if + * they have different read times. + */ private applyDocuments( documentBuffer: RemoteDocumentChangeBuffer, txn: PersistenceTransaction, @@ -727,6 +745,10 @@ export class LocalStore { }); } + /** + * Returns a promise of a `Bundle` associated with given bundle id. Promise + * resolves to undefined if no persisted data can be found. + */ getBundle(bundleId: string): Promise { return this.persistence.runTransaction( 'Get bundle', @@ -737,6 +759,10 @@ export class LocalStore { ); } + /** + * Saves the given `BundleMetadata` to local persistence. + * @param bundleMetadata + */ saveBundle(bundleMetadata: bundleProto.BundleMetadata): Promise { return this.persistence.runTransaction( 'Save bundle', @@ -747,6 +773,10 @@ export class LocalStore { ); } + /** + * Returns a promise of a `NamedQuery` associated with given query name. Promise + * resolves to undefined if no persisted data can be found. + */ getNamedQuery(queryName: string): Promise { return this.persistence.runTransaction( 'Get named query', @@ -757,6 +787,9 @@ export class LocalStore { ); } + /** + * Saves the given `NamedQuery` to local persistence. + */ saveNamedQuery(query: bundleProto.NamedQuery): Promise { return this.persistence.runTransaction( 'Save named query', diff --git a/packages/firestore/src/local/remote_document_change_buffer.ts b/packages/firestore/src/local/remote_document_change_buffer.ts index 747cd35bff6..6c0e050ee65 100644 --- a/packages/firestore/src/local/remote_document_change_buffer.ts +++ b/packages/firestore/src/local/remote_document_change_buffer.ts @@ -25,9 +25,14 @@ import { PersistenceTransaction } from './persistence'; import { PersistencePromise } from './persistence_promise'; import { SnapshotVersion } from '../core/snapshot_version'; +/** + * Represents a document change to be applied to remote document cache. + */ class RemoteDocumentChange { constructor( + // The document in this change, null if it is a removal from the cache. readonly maybeDoc: MaybeDocument | null, + // The timestamp when this change is read. readonly readTime?: SnapshotVersion ) {} } From af097c55613e3b5a3432e3f01815107e0dd76e8d Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 2 Jun 2020 17:27:01 -0400 Subject: [PATCH 18/43] Change localstore to check for newer bundle directly. --- packages/firestore/src/local/local_store.ts | 30 +++++++++--------- .../test/unit/local/local_store.test.ts | 31 ++++++++++++++++++- packages/firestore/test/util/helpers.ts | 17 ++++++++++ 3 files changed, 62 insertions(+), 16 deletions(-) diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 4d843e03bf3..c2005122726 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -70,12 +70,7 @@ import { IndexedDbTargetCache } from './indexeddb_target_cache'; import { extractFieldMask } from '../model/object_value'; import { isIndexedDbTransactionError } from './simple_db'; import * as bundleProto from '../protos/firestore_bundle_proto'; -import { - Bundle, - BundleConverter, - BundledDocuments, - NamedQuery -} from '../core/bundle'; +import { BundleConverter, BundledDocuments, NamedQuery } from '../core/bundle'; import { BundleCache } from './bundle_cache'; const LOG_TAG = 'LocalStore'; @@ -746,17 +741,22 @@ export class LocalStore { } /** - * Returns a promise of a `Bundle` associated with given bundle id. Promise - * resolves to undefined if no persisted data can be found. + * Returns a promise of a boolean to indicate if the given bundle has already + * been loaded and the create time is newer than the current loading bundle. */ - getBundle(bundleId: string): Promise { - return this.persistence.runTransaction( - 'Get bundle', - 'readonly', - transaction => { - return this.bundleCache.getBundle(transaction, bundleId); - } + isNewerBundleLoaded( + bundleMetadata: bundleProto.BundleMetadata + ): Promise { + const currentReadTime = this.bundleConverter.toSnapshotVersion( + bundleMetadata.createTime! ); + return this.persistence + .runTransaction('isNewerBundleLoaded', 'readonly', transaction => { + return this.bundleCache.getBundle(transaction, bundleMetadata.id!); + }) + .then(cached => { + return !!cached && cached!.createTime!.compareTo(currentReadTime) > 0; + }); } /** diff --git a/packages/firestore/test/unit/local/local_store.test.ts b/packages/firestore/test/unit/local/local_store.test.ts index 6b197525897..99d8b49d299 100644 --- a/packages/firestore/test/unit/local/local_store.test.ts +++ b/packages/firestore/test/unit/local/local_store.test.ts @@ -79,7 +79,8 @@ import { TestSnapshotVersion, transformMutation, unknownDoc, - version + version, + bundleMetadata } from '../../util/helpers'; import { CountingQueryEngine, QueryEngineType } from './counting_query_engine'; @@ -87,6 +88,7 @@ import * as persistenceHelpers from './persistence_test_helpers'; import { ByteString } from '../../../src/util/byte_string'; import { BundleConverter, BundledDocuments } from '../../../src/core/bundle'; import { JSON_SERIALIZER } from './persistence_test_helpers'; +import { BundleMetadata } from '../../../src/protos/firestore_bundle_proto'; export interface LocalStoreComponents { queryEngine: CountingQueryEngine; @@ -411,6 +413,25 @@ class LocalStoreTester { return this; } + toHaveNewerBundle( + metadata: BundleMetadata, + expected: boolean + ): LocalStoreTester { + this.promiseChain = this.promiseChain.then(() => { + return this.localStore.isNewerBundleLoaded(metadata).then(actual => { + expect(actual).to.equal(expected); + }); + }); + return this; + } + + afterSavingBundle(metadata: BundleMetadata): LocalStoreTester { + this.promiseChain = this.promiseChain.then(() => { + return this.localStore.saveBundle(metadata); + }); + return this; + } + finish(): Promise { return this.promiseChain; } @@ -1615,6 +1636,14 @@ function genericLocalStoreTests( .finish(); }); + it('handles saving and checking bundle metadata', () => { + return expectLocalStore() + .toHaveNewerBundle(bundleMetadata('test', 2), false) + .afterSavingBundle(bundleMetadata('test', 2)) + .toHaveNewerBundle(bundleMetadata('test', 1), true) + .finish(); + }); + it('computes highest unacknowledged batch id correctly', () => { return expectLocalStore() .toReturnHighestUnacknowledgeBatchId(BATCHID_UNKNOWN) diff --git a/packages/firestore/test/util/helpers.ts b/packages/firestore/test/util/helpers.ts index 58d0620e647..a92a1e2505f 100644 --- a/packages/firestore/test/util/helpers.ts +++ b/packages/firestore/test/util/helpers.ts @@ -94,6 +94,7 @@ import { DeleteFieldValueImpl } from '../../src/api/field_value'; import { Code, FirestoreError } from '../../src/util/error'; import { JSON_SERIALIZER } from '../unit/local/persistence_test_helpers'; import { BundledDocuments } from '../../src/core/bundle'; +import { BundleMetadata } from '../../src/protos/firestore_bundle_proto'; /* eslint-disable no-restricted-globals */ @@ -436,6 +437,22 @@ export function bundledDocuments( return new TestBundledDocuments(result); } +export function bundleMetadata( + id: string, + createTime: TestSnapshotVersion, + version = 1, + totalDocuments = 1, + totalBytes = 1000 +): BundleMetadata { + return { + id, + createTime: { seconds: createTime, nanos: 0 }, + version, + totalDocuments, + totalBytes + }; +} + export function updateMapping( snapshotVersion: SnapshotVersion, added: Array, From e735e238e01e7e587835309c0ac8dac94f1944e7 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Wed, 3 Jun 2020 08:49:32 -0400 Subject: [PATCH 19/43] temp checkin --- packages/firestore/src/core/sync_engine.ts | 26 ++++++++++++++++++++ packages/firestore/src/util/bundle_reader.ts | 4 +++ 2 files changed, 30 insertions(+) diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 0204f403e50..236efd80658 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -74,6 +74,7 @@ import { import { ViewSnapshot } from './view_snapshot'; import { AsyncQueue, wrapInUserErrorIfRecoverable } from '../util/async_queue'; import { TransactionRunner } from './transaction_runner'; +import { BundleReader } from '../util/bundle_reader'; const LOG_TAG = 'SyncEngine'; @@ -439,6 +440,31 @@ export class SyncEngine implements RemoteSyncer { } } + async loadBundle(bundleReader: BundleReader): Promise { + this.assertSubscribed('loadBundle()'); + const metadata = await bundleReader.getMetadata(); + + const skip = await this.localStore.isNewerBundleLoaded(metadata); + if (skip) { + return bundleReader.close(); + } + + while (true) { + const e = await bundleReader.nextElement(); + if (!e) { + break; + } + if (e.payload.namedQuery) { + await this.localStore.saveNamedQuery(e.payload.namedQuery); + } + if (e.payload.documentMetadata) { + await this.localStore.applyBundledDocuments(); + } + } + + return this.localStore.saveBundle(metadata); + } + /** * Applies an OnlineState change to the sync engine and notifies any views of * the change. diff --git a/packages/firestore/src/util/bundle_reader.ts b/packages/firestore/src/util/bundle_reader.ts index aff7a25c77d..d7da86a13c7 100644 --- a/packages/firestore/src/util/bundle_reader.ts +++ b/packages/firestore/src/util/bundle_reader.ts @@ -84,6 +84,10 @@ export class BundleReader { ); } + close(): Promise { + return this.reader.cancel(); + } + /** * Returns the metadata of the bundle. */ From 17ba434fa246fe3630071b658a59ddd25c91f6fb Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 4 Jun 2020 13:03:46 -0400 Subject: [PATCH 20/43] Implement without async/await --- packages/firestore/src/core/bundle.ts | 213 ++++++++++++++++++ .../firestore/src/core/firestore_client.ts | 6 + packages/firestore/src/core/sync_engine.ts | 64 ++++-- 3 files changed, 265 insertions(+), 18 deletions(-) diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index bcd8bedcfe0..0607ec52c79 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -23,6 +23,10 @@ import * as api from '../protos/firestore_proto_api'; import { DocumentKey } from '../model/document_key'; import { MaybeDocument, NoDocument } from '../model/document'; import { debugAssert } from '../util/assert'; +import { LocalStore } from '../local/local_store'; +import { SizedBundleElement } from '../util/bundle_reader'; +import { MaybeDocumentMap } from '../model/collections'; +import { Deferred } from '../util/promise'; /** * Represents a Firestore bundle saved by the SDK in its local storage. @@ -74,3 +78,212 @@ export class BundleConverter { return this.serializer.fromVersion(time); } } + +export interface LoadBundleTaskProgress { + documentsLoaded: number; + totalDocuments: number; + bytesLoaded: number; + totalBytes: number; + taskState: TaskState; +} +export type TaskState = 'Error' | 'Running' | 'Success'; + +export interface LoadBundleTask { + onProgress( + next?: (a: LoadBundleTaskProgress) => any, + error?: (a: Error) => any, + complete?: () => void + ): Promise; + + then( + onFulfilled?: (a: LoadBundleTaskProgress) => any, + onRejected?: (a: Error) => any + ): Promise; + + catch(onRejected: (a: Error) => any): Promise; +} + +export class LoadBundleTaskImpl implements LoadBundleTask { + private progressResolver = new Deferred(); + private progressNext?: (a: LoadBundleTaskProgress) => any; + private progressError?: (a: Error) => any; + private progressComplete?: () => any; + + private promiseResolver = new Deferred(); + private promiseFulfilled?: (a: LoadBundleTaskProgress) => any; + private promiseRejected?: (a: Error) => any; + + private lastProgress: LoadBundleTaskProgress = { + taskState: 'Running', + totalBytes: 0, + totalDocuments: 0, + bytesLoaded: 0, + documentsLoaded: 0 + }; + + onProgress( + next?: (a: LoadBundleTaskProgress) => any, + error?: (a: Error) => any, + complete?: () => void + ): Promise { + this.progressNext = next; + this.progressError = error; + this.progressComplete = complete; + return this.progressResolver.promise; + } + + catch(onRejected: (a: Error) => any): Promise { + this.promiseRejected = onRejected; + return this.promiseResolver.promise; + } + + then( + onFulfilled?: (a: LoadBundleTaskProgress) => any, + onRejected?: (a: Error) => any + ): Promise { + this.promiseFulfilled = onFulfilled; + this.promiseRejected = onRejected; + return this.promiseResolver.promise; + } + + completeWith(progress: LoadBundleTaskProgress): void { + let result; + if (this.progressComplete) { + result = this.progressComplete(); + } + this.progressResolver.resolve(result); + + result = undefined; + if (this.promiseFulfilled) { + result = this.promiseFulfilled(progress); + } + this.promiseResolver.resolve(result); + } + + failedWith(error: Error): void { + if (this.progressNext) { + this.lastProgress!.taskState = 'Error'; + this.progressNext(this.lastProgress); + } + + let result; + if (this.progressError) { + result = this.progressError(error); + } + this.progressResolver.reject(result); + + result = undefined; + if (this.promiseRejected) { + this.promiseRejected(error); + } + this.promiseResolver.reject(result); + } + + updateProgress(progress: LoadBundleTaskProgress): void { + this.lastProgress = progress; + if (this.progressNext) { + this.progressNext(progress); + } + } +} + +export class LoadResult { + constructor( + readonly progress: LoadBundleTaskProgress, + readonly changedDocs?: MaybeDocumentMap + ) {} +} + +export class BundleLoader { + private progress: LoadBundleTaskProgress; + private step = 0.01; + private queries: bundleProto.NamedQuery[] = []; + private documents: BundledDocuments = []; + private bytesIncrement = 0; + private documentsIncrement = 0; + private unpairedDocumentMetadata: bundleProto.BundledDocumentMetadata | null = null; + + constructor( + private metadata: bundleProto.BundleMetadata, + private localStore: LocalStore + ) { + this.progress = { + documentsLoaded: 0, + totalDocuments: metadata.totalDocuments!, + bytesLoaded: 0, + totalBytes: metadata.totalBytes!, + taskState: 'Running' + }; + } + + addSizedElement(element: SizedBundleElement): Promise { + debugAssert(!element.isBundleMetadata(), 'Unexpected bundle metadata.'); + + this.bytesIncrement += element.byteLength; + if (element.payload.namedQuery) { + this.queries.push(element.payload.namedQuery); + } + + if (element.payload.documentMetadata) { + if (element.payload.documentMetadata.exists) { + this.unpairedDocumentMetadata = element.payload.documentMetadata; + } else { + this.documents.push([element.payload.documentMetadata, undefined]); + this.documentsIncrement += 1; + } + } + + if (element.payload.document) { + debugAssert( + !this.unpairedDocumentMetadata, + 'Unexpected document when no paring metadata is found' + ); + this.documents.push([ + this.unpairedDocumentMetadata!, + element.payload.document + ]); + this.documentsIncrement += 1; + this.unpairedDocumentMetadata = null; + } + + return this.saveAndReportProgress(); + } + + private async saveAndReportProgress(): Promise { + if ( + this.unpairedDocumentMetadata || + (this.documentsIncrement < this.progress.totalDocuments * this.step && + this.bytesIncrement < this.progress.totalBytes * this.step) + ) { + return null; + } + + for (const q of this.queries) { + await this.localStore.saveNamedQuery(q); + } + + let changedDocs; + if (this.documents.length > 0) { + changedDocs = await this.localStore.applyBundledDocuments(this.documents); + } + + this.progress.bytesLoaded += this.bytesIncrement; + this.progress.documentsLoaded += this.documentsIncrement; + this.bytesIncrement = 0; + this.documentsIncrement = 0; + this.queries = []; + this.documents = []; + + return new LoadResult({ ...this.progress }, changedDocs); + } + + async complete(): Promise { + debugAssert( + this.queries.length === 0 && this.documents.length === 0, + 'There are more items needs to be saved but complete() is called.' + ); + this.progress.taskState = 'Success'; + + return this.progress; + } +} diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index 42fbe0173f5..166c4105b34 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -48,6 +48,8 @@ import { ComponentProvider, MemoryComponentProvider } from './component_provider'; +import { BundleReader } from '../util/bundle_reader'; +import { LoadBundleTask } from './bundle'; const LOG_TAG = 'FirestoreClient'; const MAX_CONCURRENT_LIMBO_RESOLUTIONS = 100; @@ -521,4 +523,8 @@ export class FirestoreClient { }); return deferred.promise; } + + loadBundle(bundleReader: BundleReader): LoadBundleTask { + this.verifyNotTerminated(); + } } diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 236efd80658..967f335836f 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -75,6 +75,8 @@ import { ViewSnapshot } from './view_snapshot'; import { AsyncQueue, wrapInUserErrorIfRecoverable } from '../util/async_queue'; import { TransactionRunner } from './transaction_runner'; import { BundleReader } from '../util/bundle_reader'; +import { BundleLoader, LoadBundleTask, LoadBundleTaskImpl } from './bundle'; +import { BundleMetadata } from '../protos/firestore_bundle_proto'; const LOG_TAG = 'SyncEngine'; @@ -440,29 +442,55 @@ export class SyncEngine implements RemoteSyncer { } } - async loadBundle(bundleReader: BundleReader): Promise { + loadBundle(bundleReader: BundleReader, task: LoadBundleTaskImpl): void { this.assertSubscribed('loadBundle()'); - const metadata = await bundleReader.getMetadata(); - - const skip = await this.localStore.isNewerBundleLoaded(metadata); - if (skip) { - return bundleReader.close(); - } + let metadata: BundleMetadata; + let loader: BundleLoader; + bundleReader + .getMetadata() + .then(m => { + metadata = m; + return this.localStore.isNewerBundleLoaded(metadata); + }) + .then(skip => { + if (skip) { + return bundleReader.close().then(() => { + // task.completeWith({}); + }); + } else { + loader = new BundleLoader(metadata, this.localStore); + return this.loadRestElements(loader, bundleReader, task) + .then(() => this.localStore.saveBundle(metadata)) + .then(() => loader.complete()) + .then(progress => task.completeWith(progress)); + } + }) + .catch(reason => { + task.failedWith(new Error(reason)); + }); + } - while (true) { - const e = await bundleReader.nextElement(); - if (!e) { - break; - } - if (e.payload.namedQuery) { - await this.localStore.saveNamedQuery(e.payload.namedQuery); + private async loadRestElements( + loader: BundleLoader, + reader: BundleReader, + task: LoadBundleTaskImpl + ): Promise { + const element = await reader.nextElement(); + if (element) { + debugAssert( + !element.payload.metadata, + 'Unexpected BundleMetadata element.' + ); + const result = await loader.addSizedElement(element); + if (result) { + task.updateProgress(result.progress); } - if (e.payload.documentMetadata) { - await this.localStore.applyBundledDocuments(); + if (result && result.changedDocs) { + this.emitNewSnapsAndNotifyLocalStore(result.changedDocs); } - } - return this.localStore.saveBundle(metadata); + return this.loadRestElements(loader, reader, task); + } } /** From f808d8d7729896e9fff56d9e6f0e849bb8c67d1c Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 4 Jun 2020 13:39:41 -0400 Subject: [PATCH 21/43] Major code complete. --- packages/firestore-types/index.d.ts | 5 ++ packages/firestore/src/api/database.ts | 7 ++ packages/firestore/src/core/bundle.ts | 2 + .../firestore/src/core/firestore_client.ts | 14 +++- packages/firestore/src/core/sync_engine.ts | 66 ++++++++++--------- 5 files changed, 60 insertions(+), 34 deletions(-) diff --git a/packages/firestore-types/index.d.ts b/packages/firestore-types/index.d.ts index 4b5e745f70a..234f4735c80 100644 --- a/packages/firestore-types/index.d.ts +++ b/packages/firestore-types/index.d.ts @@ -16,6 +16,7 @@ */ import { FirebaseApp, FirebaseNamespace } from '@firebase/app-types'; +import { LoadBundleTask } from '../firestore/src/core/bundle'; export type DocumentData = { [field: string]: any }; @@ -85,6 +86,10 @@ export class FirebaseFirestore { terminate(): Promise; + loadBundle( + bundleData: ArrayBuffer | ReadableStream /*| string */ + ): LoadBundleTask; + INTERNAL: { delete: () => Promise }; } diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index 01b58340ca1..ee38a3747ca 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -92,6 +92,7 @@ import { fieldPathFromArgument, UserDataReader } from './user_data_reader'; import { UserDataWriter } from './user_data_writer'; import { FirebaseAuthInternalName } from '@firebase/auth-interop-types'; import { Provider } from '@firebase/component'; +import { LoadBundleTask } from '../core/bundle'; // settings() defaults: const DEFAULT_HOST = 'firestore.googleapis.com'; @@ -476,6 +477,12 @@ export class Firestore implements firestore.FirebaseFirestore, FirebaseService { }; } + loadBundle( + bundleData: ArrayBuffer | ReadableStream /*| string */ + ): LoadBundleTask { + return this._firestoreClient!.loadBundle(bundleData); + } + ensureClientConfigured(): FirestoreClient { if (!this._firestoreClient) { // Kick off starting the client but don't actually wait for it. diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index 0607ec52c79..d4901f40c73 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -88,6 +88,7 @@ export interface LoadBundleTaskProgress { } export type TaskState = 'Error' | 'Running' | 'Success'; +/* eslint-disable @typescript-eslint/no-explicit-any */ export interface LoadBundleTask { onProgress( next?: (a: LoadBundleTaskProgress) => any, @@ -186,6 +187,7 @@ export class LoadBundleTaskImpl implements LoadBundleTask { } } } +/* eslint-enable @typescript-eslint/no-explicit-any */ export class LoadResult { constructor( diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index 166c4105b34..0723ec9c7bc 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -49,7 +49,7 @@ import { MemoryComponentProvider } from './component_provider'; import { BundleReader } from '../util/bundle_reader'; -import { LoadBundleTask } from './bundle'; +import { LoadBundleTask, LoadBundleTaskImpl } from './bundle'; const LOG_TAG = 'FirestoreClient'; const MAX_CONCURRENT_LIMBO_RESOLUTIONS = 100; @@ -524,7 +524,17 @@ export class FirestoreClient { return deferred.promise; } - loadBundle(bundleReader: BundleReader): LoadBundleTask { + loadBundle( + data: ReadableStream | ArrayBuffer | Uint8Array + ): LoadBundleTask { this.verifyNotTerminated(); + + const reader = new BundleReader(data); + const task = new LoadBundleTaskImpl(); + this.asyncQueue.enqueueAndForget(() => { + return this.syncEngine.loadBundle(reader, task); + }); + + return task; } } diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 967f335836f..c6956805017 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -75,8 +75,7 @@ import { ViewSnapshot } from './view_snapshot'; import { AsyncQueue, wrapInUserErrorIfRecoverable } from '../util/async_queue'; import { TransactionRunner } from './transaction_runner'; import { BundleReader } from '../util/bundle_reader'; -import { BundleLoader, LoadBundleTask, LoadBundleTaskImpl } from './bundle'; -import { BundleMetadata } from '../protos/firestore_bundle_proto'; +import { BundleLoader, LoadBundleTaskImpl } from './bundle'; const LOG_TAG = 'SyncEngine'; @@ -442,41 +441,38 @@ export class SyncEngine implements RemoteSyncer { } } - loadBundle(bundleReader: BundleReader, task: LoadBundleTaskImpl): void { + loadBundle( + bundleReader: BundleReader, + task: LoadBundleTaskImpl + ): Promise { this.assertSubscribed('loadBundle()'); - let metadata: BundleMetadata; - let loader: BundleLoader; - bundleReader - .getMetadata() - .then(m => { - metadata = m; - return this.localStore.isNewerBundleLoaded(metadata); - }) - .then(skip => { - if (skip) { - return bundleReader.close().then(() => { - // task.completeWith({}); - }); - } else { - loader = new BundleLoader(metadata, this.localStore); - return this.loadRestElements(loader, bundleReader, task) - .then(() => this.localStore.saveBundle(metadata)) - .then(() => loader.complete()) - .then(progress => task.completeWith(progress)); - } - }) - .catch(reason => { - task.failedWith(new Error(reason)); - }); + + return this.loadBundleAsync(bundleReader, task).catch(reason => { + task.failedWith(new Error(reason)); + }); } - private async loadRestElements( - loader: BundleLoader, + private async loadBundleAsync( reader: BundleReader, task: LoadBundleTaskImpl ): Promise { - const element = await reader.nextElement(); - if (element) { + const metadata = await reader.getMetadata(); + const skip = await this.localStore.isNewerBundleLoaded(metadata); + if (skip) { + await reader.close(); + task.completeWith({ + taskState: 'Success', + documentsLoaded: 0, + bytesLoaded: 0, + totalDocuments: metadata.totalDocuments!, + totalBytes: metadata.totalBytes! + }); + return; + } + + const loader = new BundleLoader(metadata, this.localStore); + let element = await reader.nextElement(); + while (element) { debugAssert( !element.payload.metadata, 'Unexpected BundleMetadata element.' @@ -486,11 +482,17 @@ export class SyncEngine implements RemoteSyncer { task.updateProgress(result.progress); } if (result && result.changedDocs) { + // eslint-disable-next-line @typescript-eslint/no-floating-promises this.emitNewSnapsAndNotifyLocalStore(result.changedDocs); } - return this.loadRestElements(loader, reader, task); + element = await reader.nextElement(); } + + await this.localStore.saveBundle(metadata); + + const completeProgress = await loader.complete(); + task.completeWith(completeProgress); } /** From db1d864a76d1a390c15f452c489429bd9503f13e Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Sat, 6 Jun 2020 18:17:54 -0400 Subject: [PATCH 22/43] Integration tests added. --- packages/firestore/src/api/database.ts | 1 + packages/firestore/src/core/bundle.ts | 40 ++- packages/firestore/src/core/sync_engine.ts | 12 +- packages/firestore/src/local/local_store.ts | 2 +- .../test/integration/api/bundle.test.ts | 227 ++++++++++++++++++ .../test/unit/local/bundle_cache.test.ts | 2 +- .../test/unit/specs/spec_test_components.ts | 4 +- .../firestore/test/unit/util/bundle.test.ts | 123 ++-------- packages/firestore/test/util/bundle_data.ts | 213 ++++++++++++++++ 9 files changed, 495 insertions(+), 129 deletions(-) create mode 100644 packages/firestore/test/integration/api/bundle.test.ts create mode 100644 packages/firestore/test/util/bundle_data.ts diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index ee38a3747ca..3a21ac61376 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -480,6 +480,7 @@ export class Firestore implements firestore.FirebaseFirestore, FirebaseService { loadBundle( bundleData: ArrayBuffer | ReadableStream /*| string */ ): LoadBundleTask { + this.ensureClientConfigured(); return this._firestoreClient!.loadBundle(bundleData); } diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index d4901f40c73..4f5995b8650 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -27,6 +27,7 @@ import { LocalStore } from '../local/local_store'; import { SizedBundleElement } from '../util/bundle_reader'; import { MaybeDocumentMap } from '../model/collections'; import { Deferred } from '../util/promise'; +import { BundleMetadata } from '../protos/firestore_bundle_proto'; /** * Represents a Firestore bundle saved by the SDK in its local storage. @@ -88,13 +89,26 @@ export interface LoadBundleTaskProgress { } export type TaskState = 'Error' | 'Running' | 'Success'; +export function initialProgress( + state: TaskState, + metadata: BundleMetadata +): LoadBundleTaskProgress { + return { + taskState: state, + documentsLoaded: state === 'Success' ? metadata.totalDocuments! : 0, + bytesLoaded: state === 'Success' ? metadata.totalBytes! : 0, + totalDocuments: metadata.totalDocuments!, + totalBytes: metadata.totalBytes! + }; +} + /* eslint-disable @typescript-eslint/no-explicit-any */ export interface LoadBundleTask { onProgress( - next?: (a: LoadBundleTaskProgress) => any, - error?: (a: Error) => any, - complete?: () => void - ): Promise; + next?: (progress: LoadBundleTaskProgress) => any, + error?: (error: Error) => any, + complete?: (progress?: LoadBundleTaskProgress) => any + ): Promise; then( onFulfilled?: (a: LoadBundleTaskProgress) => any, @@ -106,9 +120,9 @@ export interface LoadBundleTask { export class LoadBundleTaskImpl implements LoadBundleTask { private progressResolver = new Deferred(); - private progressNext?: (a: LoadBundleTaskProgress) => any; - private progressError?: (a: Error) => any; - private progressComplete?: () => any; + private progressNext?: (progress: LoadBundleTaskProgress) => any; + private progressError?: (err: Error) => any; + private progressComplete?: (progress?: LoadBundleTaskProgress) => any; private promiseResolver = new Deferred(); private promiseFulfilled?: (a: LoadBundleTaskProgress) => any; @@ -123,9 +137,9 @@ export class LoadBundleTaskImpl implements LoadBundleTask { }; onProgress( - next?: (a: LoadBundleTaskProgress) => any, - error?: (a: Error) => any, - complete?: () => void + next?: (progress: LoadBundleTaskProgress) => any, + error?: (err: Error) => any, + complete?: (progress?: LoadBundleTaskProgress) => void ): Promise { this.progressNext = next; this.progressError = error; @@ -150,7 +164,7 @@ export class LoadBundleTaskImpl implements LoadBundleTask { completeWith(progress: LoadBundleTaskProgress): void { let result; if (this.progressComplete) { - result = this.progressComplete(); + result = this.progressComplete(progress); } this.progressResolver.resolve(result); @@ -237,8 +251,8 @@ export class BundleLoader { if (element.payload.document) { debugAssert( - !this.unpairedDocumentMetadata, - 'Unexpected document when no paring metadata is found' + !!this.unpairedDocumentMetadata, + 'Unexpected document when no pairing metadata is found' ); this.documents.push([ this.unpairedDocumentMetadata!, diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index c6956805017..d40a35f5b06 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -75,7 +75,7 @@ import { ViewSnapshot } from './view_snapshot'; import { AsyncQueue, wrapInUserErrorIfRecoverable } from '../util/async_queue'; import { TransactionRunner } from './transaction_runner'; import { BundleReader } from '../util/bundle_reader'; -import { BundleLoader, LoadBundleTaskImpl } from './bundle'; +import { BundleLoader, initialProgress, LoadBundleTaskImpl } from './bundle'; const LOG_TAG = 'SyncEngine'; @@ -460,16 +460,12 @@ export class SyncEngine implements RemoteSyncer { const skip = await this.localStore.isNewerBundleLoaded(metadata); if (skip) { await reader.close(); - task.completeWith({ - taskState: 'Success', - documentsLoaded: 0, - bytesLoaded: 0, - totalDocuments: metadata.totalDocuments!, - totalBytes: metadata.totalBytes! - }); + task.completeWith(initialProgress('Success', metadata)); return; } + task.updateProgress(initialProgress('Running', metadata)); + const loader = new BundleLoader(metadata, this.localStore); let element = await reader.nextElement(); while (element) { diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index c2005122726..5468bb9e911 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -755,7 +755,7 @@ export class LocalStore { return this.bundleCache.getBundle(transaction, bundleMetadata.id!); }) .then(cached => { - return !!cached && cached!.createTime!.compareTo(currentReadTime) > 0; + return !!cached && cached!.createTime!.compareTo(currentReadTime) >= 0; }); } diff --git a/packages/firestore/test/integration/api/bundle.test.ts b/packages/firestore/test/integration/api/bundle.test.ts new file mode 100644 index 00000000000..e57a6ba50cb --- /dev/null +++ b/packages/firestore/test/integration/api/bundle.test.ts @@ -0,0 +1,227 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as firestore from '@firebase/firestore-types'; +import { expect } from 'chai'; +import { apiDescribe, toDataArray, withTestDb } from '../util/helpers'; +import { TestBundleBuilder } from '../../util/bundle_data'; +import { DatabaseId } from '../../../src/core/database_info'; +import { key } from '../../util/helpers'; +import { LoadBundleTaskProgress } from '../../../src/core/bundle'; +import { EventsAccumulator } from '../util/events_accumulator'; + +function verifySuccessProgress(p: LoadBundleTaskProgress): void { + expect(p.taskState).to.equal('Success'); + expect(p.bytesLoaded).to.be.equal(p.totalBytes); + expect(p.documentsLoaded).to.equal(p.totalDocuments); +} + +function verifyInProgress( + p: LoadBundleTaskProgress, + expectedDocuments: number +): void { + expect(p.taskState).to.equal('Running'); + expect(p.bytesLoaded).lte(p.totalBytes); + expect(p.documentsLoaded).lte(p.totalDocuments); + expect(p.documentsLoaded).to.equal(expectedDocuments); +} + +apiDescribe.only('Bundles', (persistence: boolean) => { + const encoder = new TextEncoder(); + const testDocs: { [key: string]: firestore.DocumentData } = { + a: { k: { stringValue: 'a' }, bar: { integerValue: 1 } }, + b: { k: { stringValue: 'b' }, bar: { integerValue: 2 } } + }; + + function bundleWithTestDocs( + db: firestore.FirebaseFirestore + ): TestBundleBuilder { + const a = key('coll-1/a'); + const b = key('coll-1/b'); + const builder = new TestBundleBuilder( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (db as any)._databaseId as DatabaseId + ); + builder.addDocumentMetadata(a, { seconds: 1000, nanos: 9999 }, true); + builder.addDocument( + a, + { seconds: 1, nanos: 9 }, + { seconds: 1, nanos: 9 }, + testDocs.a + ); + builder.addDocumentMetadata(b, { seconds: 1000, nanos: 9999 }, true); + builder.addDocument( + b, + { seconds: 1, nanos: 9 }, + { seconds: 1, nanos: 9 }, + testDocs.b + ); + + return builder; + } + + it('load with documents only with on progress and promise interface.', () => { + return withTestDb(persistence, async db => { + const builder = bundleWithTestDocs(db); + + const progresses: LoadBundleTaskProgress[] = []; + let completeProgress: LoadBundleTaskProgress, + fulfillProgress: LoadBundleTaskProgress; + await db + .loadBundle( + encoder.encode( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ) + ) + .onProgress( + progress => { + progresses.push(progress); + }, + err => { + throw err; + }, + progress => { + completeProgress = progress!; + return progress; + } + ) + .then(progress => { + fulfillProgress = progress; + }) + .catch(err => { + throw err; + }); + + verifySuccessProgress(completeProgress!); + verifySuccessProgress(fulfillProgress!); + expect(progresses.length).to.equal(3); + verifyInProgress(progresses[0], 0); + verifyInProgress(progresses[1], 1); + verifyInProgress(progresses[2], 2); + + // Read from cache. These documents do not exist in backend, so they can + // only be read from cache. + const snap = await db.collection('coll-1').get({ source: 'cache' }); + expect(toDataArray(snap)).to.deep.equal([ + { k: 'a', bar: 1 }, + { k: 'b', bar: 2 } + ]); + }); + }); + + it('load with documents with promise interface.', () => { + return withTestDb(persistence, async db => { + const builder = bundleWithTestDocs(db); + + let fulfillProgress: LoadBundleTaskProgress; + await db + .loadBundle( + encoder.encode( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ) + ) + .then(progress => { + fulfillProgress = progress; + }) + .catch(err => { + throw err; + }); + + verifySuccessProgress(fulfillProgress!); + + // Read from cache. These documents do not exist in backend, so they can + // only be read from cache. + const snap = await db.collection('coll-1').get({ source: 'cache' }); + expect(toDataArray(snap)).to.deep.equal([ + { k: 'a', bar: 1 }, + { k: 'b', bar: 2 } + ]); + }); + }); + + it('load for a second time skips.', () => { + return withTestDb(persistence, async db => { + const builder = bundleWithTestDocs(db); + + await db.loadBundle( + encoder.encode( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ) + ); + + let completeProgress: LoadBundleTaskProgress; + const progresses: LoadBundleTaskProgress[] = []; + await db + .loadBundle( + encoder.encode( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ) + ) + .onProgress( + progress => { + progresses.push(progress); + }, + error => {}, + progress => { + completeProgress = progress!; + } + ); + + // No loading actually happened in the second `loadBundle` call. + expect(progresses).to.be.empty; + verifySuccessProgress(completeProgress!); + + // Read from cache. These documents do not exist in backend, so they can + // only be read from cache. + const snap = await db.collection('coll-1').get({ source: 'cache' }); + expect(toDataArray(snap)).to.deep.equal([ + { k: 'a', bar: 1 }, + { k: 'b', bar: 2 } + ]); + }); + }); + + it('load with documents already pulled from backend.', () => { + return withTestDb(persistence, async db => { + await db.doc('coll-1/a').set({ k: 'a', bar: 0 }); + await db.doc('coll-1/b').set({ k: 'b', bar: 0 }); + + const accumulator = new EventsAccumulator(); + db.collection('coll-1').onSnapshot(accumulator.storeEvent); + await accumulator.awaitEvent(); + + const builder = bundleWithTestDocs(db); + const progress = await db.loadBundle( + encoder.encode( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ) + ); + + verifySuccessProgress(progress); + // The test bundle is holding ancient documents, so no events are + // generated as a result. The case where a bundle has newer doc than + // cache can only be tested in spec tests. + await accumulator.assertNoAdditionalEvents(); + + const snap = await db.collection('coll-1').get(); + expect(toDataArray(snap)).to.deep.equal([ + { k: 'a', bar: 0 }, + { k: 'b', bar: 0 } + ]); + }); + }); +}); diff --git a/packages/firestore/test/unit/local/bundle_cache.test.ts b/packages/firestore/test/unit/local/bundle_cache.test.ts index 11c7b1c7e67..c3dea781bb9 100644 --- a/packages/firestore/test/unit/local/bundle_cache.test.ts +++ b/packages/firestore/test/unit/local/bundle_cache.test.ts @@ -78,7 +78,7 @@ function genericBundleCacheTests(cacheFn: () => TestBundleCache): void { expectedQuery: Query, expectedReadSeconds: number, expectedReadNanos: number - ):void { + ): void { expect(actual.name).to.equal(expectedName); expect(actual.query.isEqual(expectedQuery)).to.be.true; expect( diff --git a/packages/firestore/test/unit/specs/spec_test_components.ts b/packages/firestore/test/unit/specs/spec_test_components.ts index 1da4d480d8c..2bf5b0883d6 100644 --- a/packages/firestore/test/unit/specs/spec_test_components.ts +++ b/packages/firestore/test/unit/specs/spec_test_components.ts @@ -51,9 +51,7 @@ import { ViewSnapshot } from '../../../src/core/view_snapshot'; import { Query } from '../../../src/core/query'; import { Mutation } from '../../../src/model/mutation'; import { expect } from 'chai'; -import { - JSON_SERIALIZER -} from '../local/persistence_test_helpers'; +import { JSON_SERIALIZER } from '../local/persistence_test_helpers'; /** * A test-only MemoryPersistence implementation that is able to inject diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index e915d2e5dfc..83aeea0c09a 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -19,12 +19,29 @@ import { BundleReader, SizedBundleElement } from '../../../src/util/bundle_reader'; -import { BundleElement } from '../../../src/protos/firestore_bundle_proto'; import { isNode } from '../../util/test_platform'; import { PlatformSupport, toByteStreamReader } from '../../../src/platform/platform'; +import { + doc1String, + doc1MetaString, + doc1Meta, + noDocMetaString, + noDocMeta, + doc2MetaString, + doc2Meta, + limitQueryString, + limitQuery, + limitToLastQuery, + limitToLastQueryString, + meta, + metaString, + doc2String, + doc1, + doc2 +} from '../../util/bundle_data'; /** * Create a `ReadableStream` from a string. @@ -51,12 +68,6 @@ export function readableStreamFromString( }); } -function lengthPrefixedString(o: {}): string { - const str = JSON.stringify(o); - const l = new TextEncoder().encode(str).byteLength; - return `${l}${str}`; -} - // Testing readableStreamFromString() is working as expected. // eslint-disable-next-line no-restricted-properties (isNode() ? describe.skip : describe)('readableStreamFromString()', () => { @@ -84,6 +95,8 @@ function lengthPrefixedString(o: {}): string { }); }); +const encoder = new TextEncoder(); + describe('Bundle ', () => { genericBundleReadingTests(1); genericBundleReadingTests(4); @@ -109,102 +122,6 @@ function genericBundleReadingTests(bytesPerRead: number): void { } } - const encoder = new TextEncoder(); - // Setting up test data. - const meta: BundleElement = { - metadata: { - id: 'test-bundle', - createTime: { seconds: 1577836805, nanos: 6 }, - version: 1, - totalDocuments: 1, - totalBytes: 416 - } - }; - const metaString = lengthPrefixedString(meta); - - const doc1Meta: BundleElement = { - documentMetadata: { - name: - 'projects/test-project/databases/(default)/documents/collectionId/doc1', - readTime: { seconds: 5, nanos: 6 }, - exists: true - } - }; - const doc1MetaString = lengthPrefixedString(doc1Meta); - const doc1: BundleElement = { - document: { - name: - 'projects/test-project/databases/(default)/documents/collectionId/doc1', - createTime: { seconds: 1, nanos: 2000000 }, - updateTime: { seconds: 3, nanos: 4000 }, - fields: { foo: { stringValue: 'value' }, bar: { integerValue: -42 } } - } - }; - const doc1String = lengthPrefixedString(doc1); - - const doc2Meta: BundleElement = { - documentMetadata: { - name: - 'projects/test-project/databases/(default)/documents/collectionId/doc2', - readTime: { seconds: 5, nanos: 6 }, - exists: true - } - }; - const doc2MetaString = lengthPrefixedString(doc2Meta); - const doc2: BundleElement = { - document: { - name: - 'projects/test-project/databases/(default)/documents/collectionId/doc2', - createTime: { seconds: 1, nanos: 2000000 }, - updateTime: { seconds: 3, nanos: 4000 }, - fields: { foo: { stringValue: 'value1' }, bar: { integerValue: 42 } } - } - }; - const doc2String = lengthPrefixedString(doc2); - - const noDocMeta: BundleElement = { - documentMetadata: { - name: - 'projects/test-project/databases/(default)/documents/collectionId/nodoc', - readTime: { seconds: 5, nanos: 6 }, - exists: false - } - }; - const noDocMetaString = lengthPrefixedString(noDocMeta); - - const limitQuery: BundleElement = { - namedQuery: { - name: 'limitQuery', - bundledQuery: { - parent: 'projects/fireeats-97d5e/databases/(default)/documents', - structuredQuery: { - from: [{ collectionId: 'node_3.7.5_7Li7XoCjutvNxwD0tpo9' }], - orderBy: [{ field: { fieldPath: 'sort' }, direction: 'DESCENDING' }], - limit: { 'value': 1 } - }, - limitType: 'FIRST' - }, - readTime: { 'seconds': 1590011379, 'nanos': 191164000 } - } - }; - const limitQueryString = lengthPrefixedString(limitQuery); - const limitToLastQuery: BundleElement = { - namedQuery: { - name: 'limitToLastQuery', - bundledQuery: { - parent: 'projects/fireeats-97d5e/databases/(default)/documents', - structuredQuery: { - from: [{ collectionId: 'node_3.7.5_7Li7XoCjutvNxwD0tpo9' }], - orderBy: [{ field: { fieldPath: 'sort' }, direction: 'ASCENDING' }], - limit: { 'value': 1 } - }, - limitType: 'LAST' - }, - readTime: { 'seconds': 1590011379, 'nanos': 543063000 } - } - }; - const limitToLastQueryString = lengthPrefixedString(limitToLastQuery); - async function getAllElements( bundle: BundleReader ): Promise { diff --git a/packages/firestore/test/util/bundle_data.ts b/packages/firestore/test/util/bundle_data.ts new file mode 100644 index 00000000000..37ea56e9662 --- /dev/null +++ b/packages/firestore/test/util/bundle_data.ts @@ -0,0 +1,213 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + BundledQuery, + BundleElement +} from '../../src/protos/firestore_bundle_proto'; +import { DatabaseId } from '../../src/core/database_info'; +import * as api from '../../src/protos/firestore_proto_api'; +import { Value } from '../../src/protos/firestore_proto_api'; +import { JsonProtoSerializer } from '../../src/remote/serializer'; +import { PlatformSupport } from '../../src/platform/platform'; +import { DocumentKey } from '../../src/model/document_key'; + +function lengthPrefixedString(o: {}): string { + const str = JSON.stringify(o); + const l = new TextEncoder().encode(str).byteLength; + return `${l}${str}`; +} + +export class TestBundleBuilder { + readonly elements: BundleElement[] = []; + private serializer: JsonProtoSerializer; + constructor(private databaseId: DatabaseId) { + this.serializer = PlatformSupport.getPlatform().newSerializer(databaseId); + } + + addDocumentMetadata( + docKey: DocumentKey, + readTime: api.Timestamp, + exists: boolean + ): TestBundleBuilder { + this.elements.push({ + documentMetadata: { + name: this.serializer.toName(docKey), + readTime, + exists + } + }); + return this; + } + addDocument( + docKey: DocumentKey, + createTime: api.Timestamp, + updateTime: api.Timestamp, + fields: api.ApiClientObjectMap + ): TestBundleBuilder { + this.elements.push({ + document: { + name: this.serializer.toName(docKey), + createTime, + updateTime, + fields + } + }); + return this; + } + addNamedQuery( + name: string, + readTime: api.Timestamp, + bundledQuery: BundledQuery + ): TestBundleBuilder { + this.elements.push({ namedQuery: { name, readTime, bundledQuery } }); + return this; + } + getMetadataElement( + id: string, + createTime: api.Timestamp, + version = 1 + ): BundleElement { + let totalDocuments = 0; + let totalBytes = 0; + const encoder = new TextEncoder(); + for (const element of this.elements) { + if (element.documentMetadata && !element.documentMetadata.exists) { + totalDocuments += 1; + } + if (element.document) { + totalDocuments += 1; + } + totalBytes += encoder.encode(lengthPrefixedString(element)).byteLength; + } + + return { + metadata: { + id, + createTime, + version, + totalDocuments, + totalBytes + } + }; + } + + build(id: string, createTime: api.Timestamp, version = 1): string { + let result = ''; + for (const element of this.elements) { + result += lengthPrefixedString(element); + } + return ( + lengthPrefixedString(this.getMetadataElement(id, createTime, version)) + + result + ); + } +} + +// Setting up test data. +export const meta: BundleElement = { + metadata: { + id: 'test-bundle', + createTime: { seconds: 1577836805, nanos: 6 }, + version: 1, + totalDocuments: 1, + totalBytes: 416 + } +}; +export const metaString = lengthPrefixedString(meta); + +export const doc1Meta: BundleElement = { + documentMetadata: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc1', + readTime: { seconds: 5, nanos: 6 }, + exists: true + } +}; +export const doc1MetaString = lengthPrefixedString(doc1Meta); +export const doc1: BundleElement = { + document: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc1', + createTime: { seconds: 1, nanos: 2000000 }, + updateTime: { seconds: 3, nanos: 4000 }, + fields: { foo: { stringValue: 'value' }, bar: { integerValue: -42 } } + } +}; +export const doc1String = lengthPrefixedString(doc1); + +export const doc2Meta: BundleElement = { + documentMetadata: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc2', + readTime: { seconds: 5, nanos: 6 }, + exists: true + } +}; +export const doc2MetaString = lengthPrefixedString(doc2Meta); +export const doc2: BundleElement = { + document: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc2', + createTime: { seconds: 1, nanos: 2000000 }, + updateTime: { seconds: 3, nanos: 4000 }, + fields: { foo: { stringValue: 'value1' }, bar: { integerValue: 42 } } + } +}; +export const doc2String = lengthPrefixedString(doc2); + +export const noDocMeta: BundleElement = { + documentMetadata: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/nodoc', + readTime: { seconds: 5, nanos: 6 }, + exists: false + } +}; +export const noDocMetaString = lengthPrefixedString(noDocMeta); + +export const limitQuery: BundleElement = { + namedQuery: { + name: 'limitQuery', + bundledQuery: { + parent: 'projects/fireeats-97d5e/databases/(default)/documents', + structuredQuery: { + from: [{ collectionId: 'node_3.7.5_7Li7XoCjutvNxwD0tpo9' }], + orderBy: [{ field: { fieldPath: 'sort' }, direction: 'DESCENDING' }], + limit: { 'value': 1 } + }, + limitType: 'FIRST' + }, + readTime: { 'seconds': 1590011379, 'nanos': 191164000 } + } +}; +export const limitQueryString = lengthPrefixedString(limitQuery); +export const limitToLastQuery: BundleElement = { + namedQuery: { + name: 'limitToLastQuery', + bundledQuery: { + parent: 'projects/fireeats-97d5e/databases/(default)/documents', + structuredQuery: { + from: [{ collectionId: 'node_3.7.5_7Li7XoCjutvNxwD0tpo9' }], + orderBy: [{ field: { fieldPath: 'sort' }, direction: 'ASCENDING' }], + limit: { 'value': 1 } + }, + limitType: 'LAST' + }, + readTime: { 'seconds': 1590011379, 'nanos': 543063000 } + } +}; +export const limitToLastQueryString = lengthPrefixedString(limitToLastQuery); From 979ffd9fc7adc7ec1a174c15c2d8e1afb96fabff Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 9 Jun 2020 00:27:45 -0400 Subject: [PATCH 23/43] Added spec tests. Two TODO: 1. load from secondary tab. 2. don't fail async queue. --- packages/firestore/src/core/sync_engine.ts | 19 +- packages/firestore/src/local/local_store.ts | 2 +- .../test/integration/api/bundle.test.ts | 25 +- .../test/unit/specs/bundle_spec.test.ts | 354 ++++++++++++++++++ .../firestore/test/unit/specs/spec_builder.ts | 8 + .../test/unit/specs/spec_test_runner.ts | 16 + 6 files changed, 418 insertions(+), 6 deletions(-) create mode 100644 packages/firestore/test/unit/specs/bundle_spec.test.ts diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index d40a35f5b06..420551d65ef 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -447,9 +447,15 @@ export class SyncEngine implements RemoteSyncer { ): Promise { this.assertSubscribed('loadBundle()'); - return this.loadBundleAsync(bundleReader, task).catch(reason => { - task.failedWith(new Error(reason)); - }); + return ( + this.loadBundleAsync(bundleReader, task) + // TODO(wuandy): Loading a bundle will fail the entire SDK if there is + // an error. We really should have a way to run operations on async queue + // and not failing the rest if there is an error. + .catch(reason => { + task.failedWith(reason); + }) + ); } private async loadBundleAsync( @@ -1323,4 +1329,11 @@ export class MultiTabSyncEngine extends SyncEngine .catch(ignoreIfPrimaryLeaseLoss); } } + + loadBundle( + bundleReader: BundleReader, + task: LoadBundleTaskImpl + ): Promise { + return super.loadBundle(bundleReader, task); + } } diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 5468bb9e911..6728bc49ae1 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -646,7 +646,7 @@ export class LocalStore { }); return this.persistence.runTransaction( 'Apply bundle documents', - 'readwrite-primary', + 'readwrite', txn => { return this.applyDocuments( documentBuffer, diff --git a/packages/firestore/test/integration/api/bundle.test.ts b/packages/firestore/test/integration/api/bundle.test.ts index e57a6ba50cb..10678ec2f00 100644 --- a/packages/firestore/test/integration/api/bundle.test.ts +++ b/packages/firestore/test/integration/api/bundle.test.ts @@ -17,7 +17,12 @@ import * as firestore from '@firebase/firestore-types'; import { expect } from 'chai'; -import { apiDescribe, toDataArray, withTestDb } from '../util/helpers'; +import { + apiDescribe, + toDataArray, + withAlternateTestDb, + withTestDb +} from '../util/helpers'; import { TestBundleBuilder } from '../../util/bundle_data'; import { DatabaseId } from '../../../src/core/database_info'; import { key } from '../../util/helpers'; @@ -40,7 +45,7 @@ function verifyInProgress( expect(p.documentsLoaded).to.equal(expectedDocuments); } -apiDescribe.only('Bundles', (persistence: boolean) => { +apiDescribe('Bundles', (persistence: boolean) => { const encoder = new TextEncoder(); const testDocs: { [key: string]: firestore.DocumentData } = { a: { k: { stringValue: 'a' }, bar: { integerValue: 1 } }, @@ -224,4 +229,20 @@ apiDescribe.only('Bundles', (persistence: boolean) => { ]); }); }); + + it('load with documents from other projects fails.', () => { + return withTestDb(persistence, async db => { + const builder = bundleWithTestDocs(db); + return withAlternateTestDb(persistence, async otherDb => { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + expect( + otherDb.loadBundle( + encoder.encode( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ) + ) + ).to.be.rejectedWith('Tried to deserialize key from different project'); + }); + }); + }); }); diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts new file mode 100644 index 00000000000..4c223179067 --- /dev/null +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -0,0 +1,354 @@ +/** + * @license + * Copyright 2017 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Query } from '../../../src/core/query'; +import { doc, path, TestSnapshotVersion, version } from '../../util/helpers'; + +import { describeSpec, specTest } from './describe_spec'; +import { client, spec } from './spec_builder'; +import { TestBundleBuilder } from '../../util/bundle_data'; +import { + JSON_SERIALIZER, + TEST_DATABASE_ID +} from '../local/persistence_test_helpers'; +import { DocumentKey } from '../../../src/model/document_key'; +import * as api from '../../../src/protos/firestore_proto_api'; +import { Value } from '../../../src/protos/firestore_proto_api'; +import { TimerId } from '../../../src/util/async_queue'; + +interface TestBundleDocument { + key: DocumentKey; + readTime: TestSnapshotVersion; + createTime?: TestSnapshotVersion; + updateTime?: TestSnapshotVersion; + content?: api.ApiClientObjectMap; +} +function bundleWithDocument(testDoc: TestBundleDocument): string { + const builder = new TestBundleBuilder(TEST_DATABASE_ID); + builder.addDocumentMetadata( + testDoc.key, + JSON_SERIALIZER.toVersion(version(testDoc.readTime)), + !!testDoc.createTime + ); + if (testDoc.createTime) { + builder.addDocument( + testDoc.key, + JSON_SERIALIZER.toVersion(version(testDoc.createTime)), + JSON_SERIALIZER.toVersion(version(testDoc.updateTime!)), + testDoc.content! + ); + } + return builder.build( + 'test-bundle', + JSON_SERIALIZER.toVersion(version(testDoc.readTime)) + ); +} + +describeSpec('Bundles:', [], () => { + specTest('Newer docs from bundles should overwrite cache.', [], () => { + const query1 = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + const docAChanged = doc('collection/a', 2999, { key: 'b' }); + + const bundleString = bundleWithDocument({ + key: docA.key, + readTime: 3000, + createTime: 1999, + updateTime: 2999, + content: { key: { stringValue: 'b' } } + }); + + return spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA) + .expectEvents(query1, { added: [docA] }) + .loadBundle(bundleString) + .expectEvents(query1, { modified: [docAChanged] }); + }); + + specTest('Newer deleted docs from bundles should delete cache.', [], () => { + const query1 = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + + const bundleString = bundleWithDocument({ + key: docA.key, + readTime: 3000 + }); + + return spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA) + .expectEvents(query1, { added: [docA] }) + .loadBundle(bundleString) + .expectEvents(query1, { removed: [docA] }); + }); + + specTest('Older deleted docs from bundles should do nothing.', [], () => { + const query1 = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + + const bundleString = bundleWithDocument({ + key: docA.key, + readTime: 999 + }); + + return ( + spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA) + .expectEvents(query1, { added: [docA] }) + // No events are expected here. + .loadBundle(bundleString) + ); + }); + + specTest( + 'Newer docs from bundles should raise snapshot only when watch catches up with acknowledged writes.', + [], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 250, { key: 'a' }); + + const bundleString1 = bundleWithDocument({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } + }); + + const bundleString2 = bundleWithDocument({ + key: docA.key, + readTime: 1001, + createTime: 250, + updateTime: 1001, + content: { key: { stringValue: 'fromBundle' } } + }); + return ( + spec() + .withGCEnabled(false) + .userListens(query) + .watchAcksFull(query, 250, docA) + .expectEvents(query, { + added: [doc('collection/a', 250, { key: 'a' })] + }) + .userPatches('collection/a', { key: 'patched' }) + .expectEvents(query, { + modified: [ + doc( + 'collection/a', + 250, + { key: 'patched' }, + { hasLocalMutations: true } + ) + ], + hasPendingWrites: true + }) + .writeAcks('collection/a', 1000) + // loading bundleString1 will not raise snapshots, because it is before + // the acknowledged mutation. + .loadBundle(bundleString1) + // loading bundleString2 will raise a snapshot, because it is after + // the acknowledged mutation. + .loadBundle(bundleString2) + .expectEvents(query, { + modified: [doc('collection/a', 1001, { key: 'fromBundle' })] + }) + ); + } + ); + + specTest( + 'Newer docs from bundles should keep not raise snapshot if there are unacknowledged writes.', + [], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 250, { key: 'a' }); + + const bundleString1 = bundleWithDocument({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } + }); + + const bundleString2 = bundleWithDocument({ + key: docA.key, + readTime: 1001, + createTime: 250, + updateTime: 1001, + content: { key: { stringValue: 'fromBundle' } } + }); + + return ( + spec() + .withGCEnabled(false) + .userListens(query) + .watchAcksFull(query, 250, docA) + .expectEvents(query, { + added: [doc('collection/a', 250, { key: 'a' })] + }) + .userPatches('collection/a', { key: 'patched' }) + .expectEvents(query, { + modified: [ + doc( + 'collection/a', + 250, + { key: 'patched' }, + { hasLocalMutations: true } + ) + ], + hasPendingWrites: true + }) + // Loading both bundles will not raise snapshots, because of the + // mutation is not acknowledged. + .loadBundle(bundleString1) + .loadBundle(bundleString2) + ); + } + ); + + specTest('Newer docs from bundles might lead to limbo doc.', [], () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + const bundleString1 = bundleWithDocument({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } + }); + + return ( + spec() + .withGCEnabled(false) + .userListens(query) + .watchAcksFull(query, 250) + // Backend tells there is no such doc. + .expectEvents(query, {}) + // Bundle tells otherwise, leads to limbo resolution. + .loadBundle(bundleString1) + .expectEvents(query, { + added: [doc('collection/a', 500, { key: 'b' })], + fromCache: true + }) + .expectLimboDocs(docA.key) + ); + }); + + specTest( + 'Load from secondary clients and observe from primary.', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 250, { key: 'a' }); + const bundleString1 = bundleWithDocument({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } + }); + + return ( + client(0) + .userListens(query) + .watchAcksFull(query, 250, docA) + .expectEvents(query, { + added: [docA] + }) + .client(1) + // Bundle tells otherwise, leads to limbo resolution. + .loadBundle(bundleString1) + .client(0) + .becomeVisible() + // TODO(wuandy): Loading from secondary client does not notify other + // clients for now. We need to fix it and uncomment below. + // .expectEvents(query, { + // modified: [doc('collection/a', 500, { key: 'b' })], + // }) + ); + } + ); + + specTest('Load and observe from secondary clients.', ['multi-client'], () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 250, { key: 'a' }); + const bundleString1 = bundleWithDocument({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } + }); + + return client(0) + .userListens(query) + .watchAcksFull(query, 250, docA) + .expectEvents(query, { + added: [docA] + }) + .client(1) + .userListens(query) + .expectEvents(query, { + added: [docA] + }) + .loadBundle(bundleString1) + .expectEvents(query, { + modified: [doc('collection/a', 500, { key: 'b' })] + }); + }); + + specTest( + 'Load from primary client and observe from secondary.', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 250, { key: 'a' }); + const bundleString1 = bundleWithDocument({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } + }); + + return ( + client(0) + .userListens(query) + .watchAcksFull(query, 250, docA) + .expectEvents(query, { + added: [docA] + }) + .client(1) + .stealPrimaryLease() + .expectListen(query, 'resume-token-250') + // Bundle tells otherwise, leads to limbo resolution. + .loadBundle(bundleString1) + .client(0) + .runTimer(TimerId.ClientMetadataRefresh) + // Client 0 recovers from its lease loss and applies the updates from + // client 1 + .expectPrimaryState(false) + .expectEvents(query, { + modified: [doc('collection/a', 500, { key: 'b' })] + }) + ); + } + ); +}); diff --git a/packages/firestore/test/unit/specs/spec_builder.ts b/packages/firestore/test/unit/specs/spec_builder.ts index d5c0f2460ef..cf728c675a0 100644 --- a/packages/firestore/test/unit/specs/spec_builder.ts +++ b/packages/firestore/test/unit/specs/spec_builder.ts @@ -345,6 +345,14 @@ export class SpecBuilder { return this; } + loadBundle(bundleContent: string): this { + this.nextStep(); + this.currentStep = { + loadBundle: bundleContent + }; + return this; + } + // PORTING NOTE: Only used by web multi-tab tests. becomeHidden(): this { this.nextStep(); diff --git a/packages/firestore/test/unit/specs/spec_test_runner.ts b/packages/firestore/test/unit/specs/spec_test_runner.ts index 26c6a284636..1ad9c3cf685 100644 --- a/packages/firestore/test/unit/specs/spec_test_runner.ts +++ b/packages/firestore/test/unit/specs/spec_test_runner.ts @@ -113,6 +113,8 @@ import { SharedWriteTracker } from './spec_test_components'; import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; +import { BundleReader } from '../../../src/util/bundle_reader'; +import { LoadBundleTaskImpl } from '../../../src/core/bundle'; const ARBITRARY_SEQUENCE_NUMBER = 2; @@ -301,6 +303,8 @@ abstract class TestRunner { return this.doAddSnapshotsInSyncListener(); } else if ('removeSnapshotsInSyncListener' in step) { return this.doRemoveSnapshotsInSyncListener(); + } else if ('loadBundle' in step) { + return this.doLoadBundle(step.loadBundle!); } else if ('watchAck' in step) { return this.doWatchAck(step.watchAck!); } else if ('watchCurrent' in step) { @@ -438,6 +442,16 @@ abstract class TestRunner { return Promise.resolve(); } + private async doLoadBundle(bundle: string): Promise { + const reader = new BundleReader(new TextEncoder().encode(bundle)); + const task = new LoadBundleTaskImpl(); + this.queue.enqueueAndForget(() => { + return this.syncEngine.loadBundle(reader, task); + }); + + await task; + } + private doMutations(mutations: Mutation[]): Promise { const documentKeys = mutations.map(val => val.key.path.toString()); const syncEngineCallback = new Deferred(); @@ -1249,6 +1263,8 @@ export interface SpecStep { addSnapshotsInSyncListener?: true; /** Unlistens from a SnapshotsInSync event. */ removeSnapshotsInSyncListener?: true; + /** Loads a bundle from a string. */ + loadBundle?: string; /** Ack for a query in the watch stream */ watchAck?: SpecWatchAck; From f7ff495d7f130d8d44b1b0ae8c60541e241b22e2 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 9 Jun 2020 08:41:35 -0400 Subject: [PATCH 24/43] Add comments and move types to d.ts --- packages/firestore-types/index.d.ts | 28 ++++- packages/firestore/src/api/database.ts | 3 +- packages/firestore/src/core/bundle.ts | 117 ++++++++++-------- .../firestore/src/core/firestore_client.ts | 3 +- packages/firestore/src/core/sync_engine.ts | 18 +-- .../test/integration/api/bundle.test.ts | 17 ++- .../test/unit/specs/bundle_spec.test.ts | 2 +- packages/firestore/test/util/bundle_data.ts | 2 +- 8 files changed, 113 insertions(+), 77 deletions(-) diff --git a/packages/firestore-types/index.d.ts b/packages/firestore-types/index.d.ts index 234f4735c80..5e315240c63 100644 --- a/packages/firestore-types/index.d.ts +++ b/packages/firestore-types/index.d.ts @@ -16,7 +16,6 @@ */ import { FirebaseApp, FirebaseNamespace } from '@firebase/app-types'; -import { LoadBundleTask } from '../firestore/src/core/bundle'; export type DocumentData = { [field: string]: any }; @@ -87,12 +86,37 @@ export class FirebaseFirestore { terminate(): Promise; loadBundle( - bundleData: ArrayBuffer | ReadableStream /*| string */ + bundleData: ArrayBuffer | ReadableStream | string ): LoadBundleTask; INTERNAL: { delete: () => Promise }; } +export interface LoadBundleTask { + onProgress( + next?: (progress: LoadBundleTaskProgress) => any, + error?: (error: Error) => any, + complete?: (progress?: LoadBundleTaskProgress) => any + ): Promise; + + then( + onFulfilled?: (a: LoadBundleTaskProgress) => any, + onRejected?: (a: Error) => any + ): Promise; + + catch(onRejected: (a: Error) => any): Promise; +} + +export interface LoadBundleTaskProgress { + documentsLoaded: number; + totalDocuments: number; + bytesLoaded: number; + totalBytes: number; + taskState: TaskState; +} + +export type TaskState = 'Error' | 'Running' | 'Success'; + export class GeoPoint { constructor(latitude: number, longitude: number); diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index 3a21ac61376..06f70ecd7bc 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -92,7 +92,6 @@ import { fieldPathFromArgument, UserDataReader } from './user_data_reader'; import { UserDataWriter } from './user_data_writer'; import { FirebaseAuthInternalName } from '@firebase/auth-interop-types'; import { Provider } from '@firebase/component'; -import { LoadBundleTask } from '../core/bundle'; // settings() defaults: const DEFAULT_HOST = 'firestore.googleapis.com'; @@ -479,7 +478,7 @@ export class Firestore implements firestore.FirebaseFirestore, FirebaseService { loadBundle( bundleData: ArrayBuffer | ReadableStream /*| string */ - ): LoadBundleTask { + ): firestore.LoadBundleTask { this.ensureClientConfigured(); return this._firestoreClient!.loadBundle(bundleData); } diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index 4f5995b8650..c92e4f02e05 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -15,6 +15,8 @@ * limitations under the License. */ +import * as firestore from '@firebase/firestore-types'; + import { Query } from './query'; import { SnapshotVersion } from './snapshot_version'; import { JsonProtoSerializer } from '../remote/serializer'; @@ -80,19 +82,14 @@ export class BundleConverter { } } -export interface LoadBundleTaskProgress { - documentsLoaded: number; - totalDocuments: number; - bytesLoaded: number; - totalBytes: number; - taskState: TaskState; -} -export type TaskState = 'Error' | 'Running' | 'Success'; - +/** + * Returns a `LoadBundleTaskProgress` representing the first progress of + * loading a bundle. + */ export function initialProgress( - state: TaskState, + state: firestore.TaskState, metadata: BundleMetadata -): LoadBundleTaskProgress { +): firestore.LoadBundleTaskProgress { return { taskState: state, documentsLoaded: state === 'Success' ? metadata.totalDocuments! : 0, @@ -103,32 +100,21 @@ export function initialProgress( } /* eslint-disable @typescript-eslint/no-explicit-any */ -export interface LoadBundleTask { - onProgress( - next?: (progress: LoadBundleTaskProgress) => any, - error?: (error: Error) => any, - complete?: (progress?: LoadBundleTaskProgress) => any - ): Promise; - - then( - onFulfilled?: (a: LoadBundleTaskProgress) => any, - onRejected?: (a: Error) => any - ): Promise; - - catch(onRejected: (a: Error) => any): Promise; -} - -export class LoadBundleTaskImpl implements LoadBundleTask { +export class LoadBundleTaskImpl implements firestore.LoadBundleTask { private progressResolver = new Deferred(); - private progressNext?: (progress: LoadBundleTaskProgress) => any; + private progressNext?: (progress: firestore.LoadBundleTaskProgress) => any; private progressError?: (err: Error) => any; - private progressComplete?: (progress?: LoadBundleTaskProgress) => any; + private progressComplete?: ( + progress?: firestore.LoadBundleTaskProgress + ) => any; private promiseResolver = new Deferred(); - private promiseFulfilled?: (a: LoadBundleTaskProgress) => any; - private promiseRejected?: (a: Error) => any; + private promiseFulfilled?: ( + progress: firestore.LoadBundleTaskProgress + ) => any; + private promiseRejected?: (err: Error) => any; - private lastProgress: LoadBundleTaskProgress = { + private lastProgress: firestore.LoadBundleTaskProgress = { taskState: 'Running', totalBytes: 0, totalDocuments: 0, @@ -137,9 +123,9 @@ export class LoadBundleTaskImpl implements LoadBundleTask { }; onProgress( - next?: (progress: LoadBundleTaskProgress) => any, + next?: (progress: firestore.LoadBundleTaskProgress) => any, error?: (err: Error) => any, - complete?: (progress?: LoadBundleTaskProgress) => void + complete?: (progress?: firestore.LoadBundleTaskProgress) => void ): Promise { this.progressNext = next; this.progressError = error; @@ -153,15 +139,20 @@ export class LoadBundleTaskImpl implements LoadBundleTask { } then( - onFulfilled?: (a: LoadBundleTaskProgress) => any, + onFulfilled?: (a: firestore.LoadBundleTaskProgress) => any, onRejected?: (a: Error) => any ): Promise { this.promiseFulfilled = onFulfilled; this.promiseRejected = onRejected; return this.promiseResolver.promise; } + /* eslint-enable @typescript-eslint/no-explicit-any */ - completeWith(progress: LoadBundleTaskProgress): void { + /** + * Notifies the completion of loading a bundle, with a provided + * `LoadBundleTaskProgress` object. + */ + completeWith(progress: firestore.LoadBundleTaskProgress): void { let result; if (this.progressComplete) { result = this.progressComplete(progress); @@ -175,9 +166,13 @@ export class LoadBundleTaskImpl implements LoadBundleTask { this.promiseResolver.resolve(result); } + /** + * Notifies a failure of loading a bundle, with a provided `Error` + * as the reason. + */ failedWith(error: Error): void { if (this.progressNext) { - this.lastProgress!.taskState = 'Error'; + this.lastProgress.taskState = 'Error'; this.progressNext(this.lastProgress); } @@ -194,44 +189,65 @@ export class LoadBundleTaskImpl implements LoadBundleTask { this.promiseResolver.reject(result); } - updateProgress(progress: LoadBundleTaskProgress): void { + /** + * Notifies a progress update of loading a bundle. + * @param progress The new progress. + */ + updateProgress(progress: firestore.LoadBundleTaskProgress): void { this.lastProgress = progress; if (this.progressNext) { this.progressNext(progress); } } } -/* eslint-enable @typescript-eslint/no-explicit-any */ export class LoadResult { constructor( - readonly progress: LoadBundleTaskProgress, + readonly progress: firestore.LoadBundleTaskProgress, readonly changedDocs?: MaybeDocumentMap ) {} } +/** + * A class to process the elements from a bundle, load them into local + * storage and provide progress update while loading. + */ export class BundleLoader { - private progress: LoadBundleTaskProgress; + /** The current progress of loading */ + private progress: firestore.LoadBundleTaskProgress; + /** + * The threshold multiplier used to determine whether enough elements are + * batched to be loaded, and a progress update is needed. + */ private step = 0.01; + /** Batched queries to be saved into storage */ private queries: bundleProto.NamedQuery[] = []; + /** Batched documents to be saved into storage */ private documents: BundledDocuments = []; + /** How many bytes in the bundle are being batched. */ private bytesIncrement = 0; + /** How many documents in the bundle are being batched. */ private documentsIncrement = 0; + /** + * A BundleDocumentMetadata is added to the loader, it is saved here while + * we wait for the actual document. + */ private unpairedDocumentMetadata: bundleProto.BundledDocumentMetadata | null = null; constructor( private metadata: bundleProto.BundleMetadata, private localStore: LocalStore ) { - this.progress = { - documentsLoaded: 0, - totalDocuments: metadata.totalDocuments!, - bytesLoaded: 0, - totalBytes: metadata.totalBytes!, - taskState: 'Running' - }; + this.progress = initialProgress('Running', metadata); } + /** + * Adds an element from the bundle to the loader. + * + * If adding this element leads to actually saving the batched elements into + * storage, the returned promise will resolve to a `LoadResult`, otherwise + * it will resolve to null. + */ addSizedElement(element: SizedBundleElement): Promise { debugAssert(!element.isBundleMetadata(), 'Unexpected bundle metadata.'); @@ -293,7 +309,10 @@ export class BundleLoader { return new LoadResult({ ...this.progress }, changedDocs); } - async complete(): Promise { + /** + * Update the progress to 'Success' and return the updated progress. + */ + complete(): firestore.LoadBundleTaskProgress { debugAssert( this.queries.length === 0 && this.documents.length === 0, 'There are more items needs to be saved but complete() is called.' diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index 0723ec9c7bc..7ad50d227ea 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -15,6 +15,7 @@ * limitations under the License. */ +import { LoadBundleTask } from '@firebase/firestore-types'; import { CredentialsProvider } from '../api/credentials'; import { User } from '../auth/user'; import { LocalStore } from '../local/local_store'; @@ -49,7 +50,7 @@ import { MemoryComponentProvider } from './component_provider'; import { BundleReader } from '../util/bundle_reader'; -import { LoadBundleTask, LoadBundleTaskImpl } from './bundle'; +import { LoadBundleTaskImpl } from './bundle'; const LOG_TAG = 'FirestoreClient'; const MAX_CONCURRENT_LIMBO_RESOLUTIONS = 100; diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 420551d65ef..62f110beac1 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -482,10 +482,11 @@ export class SyncEngine implements RemoteSyncer { const result = await loader.addSizedElement(element); if (result) { task.updateProgress(result.progress); - } - if (result && result.changedDocs) { - // eslint-disable-next-line @typescript-eslint/no-floating-promises - this.emitNewSnapsAndNotifyLocalStore(result.changedDocs); + + if (result.changedDocs) { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + this.emitNewSnapsAndNotifyLocalStore(result.changedDocs); + } } element = await reader.nextElement(); @@ -493,7 +494,7 @@ export class SyncEngine implements RemoteSyncer { await this.localStore.saveBundle(metadata); - const completeProgress = await loader.complete(); + const completeProgress = loader.complete(); task.completeWith(completeProgress); } @@ -1329,11 +1330,4 @@ export class MultiTabSyncEngine extends SyncEngine .catch(ignoreIfPrimaryLeaseLoss); } } - - loadBundle( - bundleReader: BundleReader, - task: LoadBundleTaskImpl - ): Promise { - return super.loadBundle(bundleReader, task); - } } diff --git a/packages/firestore/test/integration/api/bundle.test.ts b/packages/firestore/test/integration/api/bundle.test.ts index 10678ec2f00..10f7f562a85 100644 --- a/packages/firestore/test/integration/api/bundle.test.ts +++ b/packages/firestore/test/integration/api/bundle.test.ts @@ -26,17 +26,16 @@ import { import { TestBundleBuilder } from '../../util/bundle_data'; import { DatabaseId } from '../../../src/core/database_info'; import { key } from '../../util/helpers'; -import { LoadBundleTaskProgress } from '../../../src/core/bundle'; import { EventsAccumulator } from '../util/events_accumulator'; -function verifySuccessProgress(p: LoadBundleTaskProgress): void { +function verifySuccessProgress(p: firestore.LoadBundleTaskProgress): void { expect(p.taskState).to.equal('Success'); expect(p.bytesLoaded).to.be.equal(p.totalBytes); expect(p.documentsLoaded).to.equal(p.totalDocuments); } function verifyInProgress( - p: LoadBundleTaskProgress, + p: firestore.LoadBundleTaskProgress, expectedDocuments: number ): void { expect(p.taskState).to.equal('Running'); @@ -83,9 +82,9 @@ apiDescribe('Bundles', (persistence: boolean) => { return withTestDb(persistence, async db => { const builder = bundleWithTestDocs(db); - const progresses: LoadBundleTaskProgress[] = []; - let completeProgress: LoadBundleTaskProgress, - fulfillProgress: LoadBundleTaskProgress; + const progresses: firestore.LoadBundleTaskProgress[] = []; + let completeProgress: firestore.LoadBundleTaskProgress, + fulfillProgress: firestore.LoadBundleTaskProgress; await db .loadBundle( encoder.encode( @@ -132,7 +131,7 @@ apiDescribe('Bundles', (persistence: boolean) => { return withTestDb(persistence, async db => { const builder = bundleWithTestDocs(db); - let fulfillProgress: LoadBundleTaskProgress; + let fulfillProgress: firestore.LoadBundleTaskProgress; await db .loadBundle( encoder.encode( @@ -168,8 +167,8 @@ apiDescribe('Bundles', (persistence: boolean) => { ) ); - let completeProgress: LoadBundleTaskProgress; - const progresses: LoadBundleTaskProgress[] = []; + let completeProgress: firestore.LoadBundleTaskProgress; + const progresses: firestore.LoadBundleTaskProgress[] = []; await db .loadBundle( encoder.encode( diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts index 4c223179067..ff4109d3c5e 100644 --- a/packages/firestore/test/unit/specs/bundle_spec.test.ts +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -241,7 +241,7 @@ describeSpec('Bundles:', [], () => { .watchAcksFull(query, 250) // Backend tells there is no such doc. .expectEvents(query, {}) - // Bundle tells otherwise, leads to limbo resolution. + // Bundle tells otherwise, leads to limbo. .loadBundle(bundleString1) .expectEvents(query, { added: [doc('collection/a', 500, { key: 'b' })], diff --git a/packages/firestore/test/util/bundle_data.ts b/packages/firestore/test/util/bundle_data.ts index 37ea56e9662..125acbb077b 100644 --- a/packages/firestore/test/util/bundle_data.ts +++ b/packages/firestore/test/util/bundle_data.ts @@ -117,7 +117,7 @@ export class TestBundleBuilder { } } -// Setting up test data. +// TODO(wuandy): Ideally, these should use `TestBundleBuilder` above. export const meta: BundleElement = { metadata: { id: 'test-bundle', From 556a0072ffaa8429b1583d7abeb80a017001aba2 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 9 Jun 2020 11:54:12 -0400 Subject: [PATCH 25/43] Support loading string for real. --- packages/firestore-types/index.d.ts | 2 +- packages/firestore/src/api/database.ts | 2 +- packages/firestore/src/core/firestore_client.ts | 10 ++++++++-- .../test/integration/api/bundle.test.ts | 17 +++++------------ 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/packages/firestore-types/index.d.ts b/packages/firestore-types/index.d.ts index 5e315240c63..7c543b6a9c6 100644 --- a/packages/firestore-types/index.d.ts +++ b/packages/firestore-types/index.d.ts @@ -86,7 +86,7 @@ export class FirebaseFirestore { terminate(): Promise; loadBundle( - bundleData: ArrayBuffer | ReadableStream | string + bundleData: ArrayBuffer | ReadableStream | string ): LoadBundleTask; INTERNAL: { delete: () => Promise }; diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index 06f70ecd7bc..248720ac8f6 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -477,7 +477,7 @@ export class Firestore implements firestore.FirebaseFirestore, FirebaseService { } loadBundle( - bundleData: ArrayBuffer | ReadableStream /*| string */ + bundleData: ArrayBuffer | ReadableStream | string ): firestore.LoadBundleTask { this.ensureClientConfigured(); return this._firestoreClient!.loadBundle(bundleData); diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index 7ad50d227ea..c69713925fe 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -526,11 +526,17 @@ export class FirestoreClient { } loadBundle( - data: ReadableStream | ArrayBuffer | Uint8Array + data: ReadableStream | ArrayBuffer | string ): LoadBundleTask { this.verifyNotTerminated(); - const reader = new BundleReader(data); + let content: ReadableStream | ArrayBuffer; + if (typeof data === 'string') { + content = new TextEncoder().encode(data); + } else { + content = data; + } + const reader = new BundleReader(content); const task = new LoadBundleTaskImpl(); this.asyncQueue.enqueueAndForget(() => { return this.syncEngine.loadBundle(reader, task); diff --git a/packages/firestore/test/integration/api/bundle.test.ts b/packages/firestore/test/integration/api/bundle.test.ts index 10f7f562a85..2d78915be2a 100644 --- a/packages/firestore/test/integration/api/bundle.test.ts +++ b/packages/firestore/test/integration/api/bundle.test.ts @@ -87,9 +87,7 @@ apiDescribe('Bundles', (persistence: boolean) => { fulfillProgress: firestore.LoadBundleTaskProgress; await db .loadBundle( - encoder.encode( - builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) - ) + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ) .onProgress( progress => { @@ -134,9 +132,7 @@ apiDescribe('Bundles', (persistence: boolean) => { let fulfillProgress: firestore.LoadBundleTaskProgress; await db .loadBundle( - encoder.encode( - builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) - ) + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ) .then(progress => { fulfillProgress = progress; @@ -162,9 +158,7 @@ apiDescribe('Bundles', (persistence: boolean) => { const builder = bundleWithTestDocs(db); await db.loadBundle( - encoder.encode( - builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) - ) + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ); let completeProgress: firestore.LoadBundleTaskProgress; @@ -210,6 +204,7 @@ apiDescribe('Bundles', (persistence: boolean) => { const builder = bundleWithTestDocs(db); const progress = await db.loadBundle( + // Testing passing non-string bundles. encoder.encode( builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ) @@ -236,9 +231,7 @@ apiDescribe('Bundles', (persistence: boolean) => { // eslint-disable-next-line @typescript-eslint/no-floating-promises expect( otherDb.loadBundle( - encoder.encode( - builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) - ) + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ) ).to.be.rejectedWith('Tried to deserialize key from different project'); }); From adf1504cbc7abcac86e24eca40a638f5823de49e Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 11 Jun 2020 11:09:34 -0400 Subject: [PATCH 26/43] Makes sure SDK still works after loading bad bundles. --- packages/firestore/src/core/sync_engine.ts | 3 --- .../test/integration/api/bundle.test.ts | 19 ++++++++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 62f110beac1..19add4ca533 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -449,9 +449,6 @@ export class SyncEngine implements RemoteSyncer { return ( this.loadBundleAsync(bundleReader, task) - // TODO(wuandy): Loading a bundle will fail the entire SDK if there is - // an error. We really should have a way to run operations on async queue - // and not failing the rest if there is an error. .catch(reason => { task.failedWith(reason); }) diff --git a/packages/firestore/test/integration/api/bundle.test.ts b/packages/firestore/test/integration/api/bundle.test.ts index 2d78915be2a..c5097fd75dd 100644 --- a/packages/firestore/test/integration/api/bundle.test.ts +++ b/packages/firestore/test/integration/api/bundle.test.ts @@ -226,7 +226,7 @@ apiDescribe('Bundles', (persistence: boolean) => { it('load with documents from other projects fails.', () => { return withTestDb(persistence, async db => { - const builder = bundleWithTestDocs(db); + let builder = bundleWithTestDocs(db); return withAlternateTestDb(persistence, async otherDb => { // eslint-disable-next-line @typescript-eslint/no-floating-promises expect( @@ -234,6 +234,23 @@ apiDescribe('Bundles', (persistence: boolean) => { builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ) ).to.be.rejectedWith('Tried to deserialize key from different project'); + + // Verify otherDb still functions, despite loaded a problematic bundle. + builder = bundleWithTestDocs(otherDb); + const progress = await otherDb.loadBundle( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ); + verifySuccessProgress(progress); + + // Read from cache. These documents do not exist in backend, so they can + // only be read from cache. + const snap = await otherDb + .collection('coll-1') + .get({ source: 'cache' }); + expect(toDataArray(snap)).to.deep.equal([ + { k: 'a', bar: 1 }, + { k: 'b', bar: 2 } + ]); }); }); }); From b364ab0e0ca8274a94269708dbf6e996f429b7c1 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 12 Jun 2020 15:45:54 -0400 Subject: [PATCH 27/43] Better spect test. --- packages/firestore/src/core/sync_engine.ts | 9 +- .../test/unit/specs/bundle_spec.test.ts | 98 ++++++++++--------- 2 files changed, 53 insertions(+), 54 deletions(-) diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 19add4ca533..c0adbf3baa7 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -447,12 +447,9 @@ export class SyncEngine implements RemoteSyncer { ): Promise { this.assertSubscribed('loadBundle()'); - return ( - this.loadBundleAsync(bundleReader, task) - .catch(reason => { - task.failedWith(reason); - }) - ); + return this.loadBundleAsync(bundleReader, task).catch(reason => { + task.failedWith(reason); + }); } private async loadBundleAsync( diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts index ff4109d3c5e..143e9838aab 100644 --- a/packages/firestore/test/unit/specs/bundle_spec.test.ts +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -28,7 +28,6 @@ import { import { DocumentKey } from '../../../src/model/document_key'; import * as api from '../../../src/protos/firestore_proto_api'; import { Value } from '../../../src/protos/firestore_proto_api'; -import { TimerId } from '../../../src/util/async_queue'; interface TestBundleDocument { key: DocumentKey; @@ -286,33 +285,37 @@ describeSpec('Bundles:', [], () => { } ); - specTest('Load and observe from secondary clients.', ['multi-client'], () => { - const query = Query.atPath(path('collection')); - const docA = doc('collection/a', 250, { key: 'a' }); - const bundleString1 = bundleWithDocument({ - key: docA.key, - readTime: 500, - createTime: 250, - updateTime: 500, - content: { key: { stringValue: 'b' } } - }); - - return client(0) - .userListens(query) - .watchAcksFull(query, 250, docA) - .expectEvents(query, { - added: [docA] - }) - .client(1) - .userListens(query) - .expectEvents(query, { - added: [docA] - }) - .loadBundle(bundleString1) - .expectEvents(query, { - modified: [doc('collection/a', 500, { key: 'b' })] + specTest( + 'Load and observe from same secondary client.', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 250, { key: 'a' }); + const bundleString1 = bundleWithDocument({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } }); - }); + + return client(0) + .userListens(query) + .watchAcksFull(query, 250, docA) + .expectEvents(query, { + added: [docA] + }) + .client(1) + .userListens(query) + .expectEvents(query, { + added: [docA] + }) + .loadBundle(bundleString1) + .expectEvents(query, { + modified: [doc('collection/a', 500, { key: 'b' })] + }); + } + ); specTest( 'Load from primary client and observe from secondary.', @@ -328,27 +331,26 @@ describeSpec('Bundles:', [], () => { content: { key: { stringValue: 'b' } } }); - return ( - client(0) - .userListens(query) - .watchAcksFull(query, 250, docA) - .expectEvents(query, { - added: [docA] - }) - .client(1) - .stealPrimaryLease() - .expectListen(query, 'resume-token-250') - // Bundle tells otherwise, leads to limbo resolution. - .loadBundle(bundleString1) - .client(0) - .runTimer(TimerId.ClientMetadataRefresh) - // Client 0 recovers from its lease loss and applies the updates from - // client 1 - .expectPrimaryState(false) - .expectEvents(query, { - modified: [doc('collection/a', 500, { key: 'b' })] - }) - ); + return client(0) + .userListens(query) + .watchAcksFull(query, 250, docA) + .expectEvents(query, { + added: [docA] + }) + .client(1) + .userListens(query) + .expectEvents(query, { + added: [docA] + }) + .client(0) + .loadBundle(bundleString1) + .expectEvents(query, { + modified: [doc('collection/a', 500, { key: 'b' })] + }) + .client(1) + .expectEvents(query, { + modified: [doc('collection/a', 500, { key: 'b' })] + }); } ); }); From 2588f200eb8cc8fd71fd3103ffff1445ec22c9a6 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 12 Jun 2020 21:32:51 -0400 Subject: [PATCH 28/43] Load bundles from secondary clients should raise snapshots --- packages/firestore/src/core/sync_engine.ts | 17 +++++++++++--- .../src/local/shared_client_state.ts | 22 +++++++++++++++++++ .../src/local/shared_client_state_schema.ts | 12 ++++++++++ .../src/local/shared_client_state_syncer.ts | 6 +++++ .../unit/local/persistence_test_helpers.ts | 2 ++ .../web_storage_shared_client_state.test.ts | 2 ++ .../test/unit/specs/bundle_spec.test.ts | 9 +++----- 7 files changed, 61 insertions(+), 9 deletions(-) diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index c0adbf3baa7..5640c604a36 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -447,9 +447,13 @@ export class SyncEngine implements RemoteSyncer { ): Promise { this.assertSubscribed('loadBundle()'); - return this.loadBundleAsync(bundleReader, task).catch(reason => { - task.failedWith(reason); - }); + return this.loadBundleAsync(bundleReader, task) + .catch(reason => { + task.failedWith(reason); + }) + .then(() => { + this.sharedClientState.remoteDocumentsChanged(); + }); } private async loadBundleAsync( @@ -1047,6 +1051,13 @@ export class MultiTabSyncEngine extends SyncEngine } } + async synchronizeWithChangedDocuments(): Promise { + if (this.isPrimaryClient) { + const changes = await this.localStore.getNewDocumentChanges(); + await this.emitNewSnapsAndNotifyLocalStore(changes); + } + } + async applyBatchState( batchId: BatchId, batchState: MutationBatchState, diff --git a/packages/firestore/src/local/shared_client_state.ts b/packages/firestore/src/local/shared_client_state.ts index dba0aa6998b..9a68e88939d 100644 --- a/packages/firestore/src/local/shared_client_state.ts +++ b/packages/firestore/src/local/shared_client_state.ts @@ -41,6 +41,7 @@ import { import { CLIENT_STATE_KEY_PREFIX, ClientStateSchema, + createRemoteDocumentsChangedKey, createWebStorageClientStateKey, createWebStorageMutationBatchKey, createWebStorageOnlineStateKey, @@ -174,6 +175,8 @@ export interface SharedClientState { setOnlineState(onlineState: OnlineState): void; writeSequenceNumber(sequenceNumber: ListenSequenceNumber): void; + + remoteDocumentsChanged(): void; } /** @@ -478,6 +481,7 @@ export class WebStorageSharedClientState implements SharedClientState { private readonly sequenceNumberKey: string; private readonly storageListener = this.handleWebStorageEvent.bind(this); private readonly onlineStateKey: string; + private readonly remoteDocumentsChangedKey: string; private readonly clientStateKeyRe: RegExp; private readonly mutationBatchKeyRe: RegExp; private readonly queryTargetKeyRe: RegExp; @@ -539,6 +543,10 @@ export class WebStorageSharedClientState implements SharedClientState { this.onlineStateKey = createWebStorageOnlineStateKey(this.persistenceKey); + this.remoteDocumentsChangedKey = createRemoteDocumentsChangedKey( + this.persistenceKey + ); + // Rather than adding the storage observer during start(), we add the // storage observer during initialization. This ensures that we collect // events before other components populate their initial state (during their @@ -718,6 +726,10 @@ export class WebStorageSharedClientState implements SharedClientState { this.persistOnlineState(onlineState); } + remoteDocumentsChanged(): void { + this.persistRemoteDocumentsChangedState(); + } + shutdown(): void { if (this.started) { this.platform.window!.removeEventListener( @@ -819,6 +831,8 @@ export class WebStorageSharedClientState implements SharedClientState { if (sequenceNumber !== ListenSequence.INVALID) { this.sequenceNumberHandler!(sequenceNumber); } + } else if (event.key === this.remoteDocumentsChangedKey) { + return this.syncEngine!.synchronizeWithChangedDocuments(); } }); } @@ -884,6 +898,10 @@ export class WebStorageSharedClientState implements SharedClientState { this.setItem(targetKey, targetMetadata.toWebStorageJSON()); } + private persistRemoteDocumentsChangedState(): void { + this.setItem(this.remoteDocumentsChangedKey, 'value-not-used'); + } + /** * Parses a client state key in WebStorage. Returns null if the key does not * match the expected key format. @@ -1132,4 +1150,8 @@ export class MemorySharedClientState implements SharedClientState { shutdown(): void {} writeSequenceNumber(sequenceNumber: ListenSequenceNumber): void {} + + remoteDocumentsChanged(): void { + // No op. + } } diff --git a/packages/firestore/src/local/shared_client_state_schema.ts b/packages/firestore/src/local/shared_client_state_schema.ts index 01665e91cfa..5d4da7f29c6 100644 --- a/packages/firestore/src/local/shared_client_state_schema.ts +++ b/packages/firestore/src/local/shared_client_state_schema.ts @@ -115,6 +115,18 @@ export function createWebStorageOnlineStateKey(persistenceKey: string): string { return `${ONLINE_STATE_KEY_PREFIX}_${persistenceKey}`; } +// The WebStorage prefix that plays as a event to indicate the remote documents +// might have changed due to some actions secondary tabs did. +// format of the key is: +// firestore_remote_documents_changed_ +export const REMOTE_DOCUMENTS_CHANGED_KEY_PREFIX = + 'firestore_remote_documents_changed'; +export function createRemoteDocumentsChangedKey( + persistenceKey: string +): string { + return `${REMOTE_DOCUMENTS_CHANGED_KEY_PREFIX}_${persistenceKey}`; +} + /** * The JSON representation of the system's online state, as written by the * primary client. diff --git a/packages/firestore/src/local/shared_client_state_syncer.ts b/packages/firestore/src/local/shared_client_state_syncer.ts index 29736807277..2ddaa991b5c 100644 --- a/packages/firestore/src/local/shared_client_state_syncer.ts +++ b/packages/firestore/src/local/shared_client_state_syncer.ts @@ -49,4 +49,10 @@ export interface SharedClientStateSyncer { /** Returns the IDs of the clients that are currently active. */ getActiveClients(): Promise; + + /** + * Query for changed documents from remote document cache and raise snapshots + * if needed. + */ + synchronizeWithChangedDocuments(): Promise; } diff --git a/packages/firestore/test/unit/local/persistence_test_helpers.ts b/packages/firestore/test/unit/local/persistence_test_helpers.ts index 9733ae9e009..9a1a688b6e2 100644 --- a/packages/firestore/test/unit/local/persistence_test_helpers.ts +++ b/packages/firestore/test/unit/local/persistence_test_helpers.ts @@ -173,6 +173,8 @@ class NoOpSharedClientStateSyncer implements SharedClientStateSyncer { removed: TargetId[] ): Promise {} applyOnlineStateChange(onlineState: OnlineState): void {} + + async synchronizeWithChangedDocuments(): Promise {} } /** * Populates Web Storage with instance data from a pre-existing client. diff --git a/packages/firestore/test/unit/local/web_storage_shared_client_state.test.ts b/packages/firestore/test/unit/local/web_storage_shared_client_state.test.ts index a9dcdf8d799..386632c2c5b 100644 --- a/packages/firestore/test/unit/local/web_storage_shared_client_state.test.ts +++ b/packages/firestore/test/unit/local/web_storage_shared_client_state.test.ts @@ -150,6 +150,8 @@ class TestSharedClientSyncer implements SharedClientStateSyncer { applyOnlineStateChange(onlineState: OnlineState): void { this.onlineState = onlineState; } + + async synchronizeWithChangedDocuments(): Promise {} } describe('WebStorageSharedClientState', () => { diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts index 143e9838aab..f6a9647c193 100644 --- a/packages/firestore/test/unit/specs/bundle_spec.test.ts +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -275,12 +275,9 @@ describeSpec('Bundles:', [], () => { // Bundle tells otherwise, leads to limbo resolution. .loadBundle(bundleString1) .client(0) - .becomeVisible() - // TODO(wuandy): Loading from secondary client does not notify other - // clients for now. We need to fix it and uncomment below. - // .expectEvents(query, { - // modified: [doc('collection/a', 500, { key: 'b' })], - // }) + .expectEvents(query, { + modified: [doc('collection/a', 500, { key: 'b' })] + }) ); } ); From 93bc9646a065196034851b69a40c7b38b2ead95a Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 12 Jun 2020 22:40:01 -0400 Subject: [PATCH 29/43] Support secondary to secondary. --- packages/firestore/src/core/sync_engine.ts | 10 ++-- .../test/unit/specs/bundle_spec.test.ts | 47 ++++++------------- 2 files changed, 19 insertions(+), 38 deletions(-) diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 5640c604a36..d9096523661 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -452,7 +452,9 @@ export class SyncEngine implements RemoteSyncer { task.failedWith(reason); }) .then(() => { - this.sharedClientState.remoteDocumentsChanged(); + if (!this.isPrimaryClient) { + this.sharedClientState.remoteDocumentsChanged(); + } }); } @@ -1052,10 +1054,8 @@ export class MultiTabSyncEngine extends SyncEngine } async synchronizeWithChangedDocuments(): Promise { - if (this.isPrimaryClient) { - const changes = await this.localStore.getNewDocumentChanges(); - await this.emitNewSnapsAndNotifyLocalStore(changes); - } + const changes = await this.localStore.getNewDocumentChanges(); + await this.emitNewSnapsAndNotifyLocalStore(changes); } async applyBatchState( diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts index f6a9647c193..428f7696519 100644 --- a/packages/firestore/test/unit/specs/bundle_spec.test.ts +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -251,39 +251,7 @@ describeSpec('Bundles:', [], () => { }); specTest( - 'Load from secondary clients and observe from primary.', - ['multi-client'], - () => { - const query = Query.atPath(path('collection')); - const docA = doc('collection/a', 250, { key: 'a' }); - const bundleString1 = bundleWithDocument({ - key: docA.key, - readTime: 500, - createTime: 250, - updateTime: 500, - content: { key: { stringValue: 'b' } } - }); - - return ( - client(0) - .userListens(query) - .watchAcksFull(query, 250, docA) - .expectEvents(query, { - added: [docA] - }) - .client(1) - // Bundle tells otherwise, leads to limbo resolution. - .loadBundle(bundleString1) - .client(0) - .expectEvents(query, { - modified: [doc('collection/a', 500, { key: 'b' })] - }) - ); - } - ); - - specTest( - 'Load and observe from same secondary client.', + 'Load and observe from secondary and observe from others.', ['multi-client'], () => { const query = Query.atPath(path('collection')); @@ -307,7 +275,20 @@ describeSpec('Bundles:', [], () => { .expectEvents(query, { added: [docA] }) + .client(2) + .userListens(query) + .expectEvents(query, { + added: [docA] + }) .loadBundle(bundleString1) + .expectEvents(query, { + modified: [doc('collection/a', 500, { key: 'b' })] + }) + .client(0) + .expectEvents(query, { + modified: [doc('collection/a', 500, { key: 'b' })] + }) + .client(1) .expectEvents(query, { modified: [doc('collection/a', 500, { key: 'b' })] }); From cb56334073fadc06cf7696ed19e914ac89a7b449 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Sat, 13 Jun 2020 20:57:49 -0400 Subject: [PATCH 30/43] Implementation. --- packages/firestore/src/api/database.ts | 17 +++++++++++++++-- packages/firestore/src/core/event_manager.ts | 18 ++++++++++++++++-- .../firestore/src/core/firestore_client.ts | 11 +++++++++-- packages/firestore/src/core/sync_engine.ts | 10 ++++++++-- packages/firestore/src/local/local_store.ts | 8 ++++++-- .../src/protos/firestore_proto_api.d.ts | 2 +- packages/firestore/src/remote/serializer.ts | 4 ++++ 7 files changed, 59 insertions(+), 11 deletions(-) diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index 248720ac8f6..49f0ae4b5c3 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -92,6 +92,7 @@ import { fieldPathFromArgument, UserDataReader } from './user_data_reader'; import { UserDataWriter } from './user_data_writer'; import { FirebaseAuthInternalName } from '@firebase/auth-interop-types'; import { Provider } from '@firebase/component'; +import { SnapshotVersion } from '../core/snapshot_version'; // settings() defaults: const DEFAULT_HOST = 'firestore.googleapis.com'; @@ -483,6 +484,16 @@ export class Firestore implements firestore.FirebaseFirestore, FirebaseService { return this._firestoreClient!.loadBundle(bundleData); } + async namedQuery(name: string): Promise { + this.ensureClientConfigured(); + const namedQuery = await this._firestoreClient!.getNamedQuery(name); + if (!namedQuery) { + return null; + } + + return new Query(namedQuery.query, this, undefined, namedQuery.readTime); + } + ensureClientConfigured(): FirestoreClient { if (!this._firestoreClient) { // Kick off starting the client but don't actually wait for it. @@ -1440,7 +1451,8 @@ export class Query implements firestore.Query { constructor( public _query: InternalQuery, readonly firestore: Firestore, - protected readonly _converter?: firestore.FirestoreDataConverter + protected readonly _converter?: firestore.FirestoreDataConverter, + readonly _readFrom?: SnapshotVersion ) {} where( @@ -1659,7 +1671,7 @@ export class Query implements firestore.Query { withConverter( converter: firestore.FirestoreDataConverter ): firestore.Query { - return new Query(this._query, this.firestore, converter); + return new Query(this._query, this.firestore, converter, this._readFrom); } /** Helper function to create a bound from a document or fields */ @@ -1904,6 +1916,7 @@ export class Query implements firestore.Query { }); const firestoreClient = this.firestore.ensureClientConfigured(); + options.readFrom = this._readFrom; const internalListener = firestoreClient.listen( this._query, asyncObserver, diff --git a/packages/firestore/src/core/event_manager.ts b/packages/firestore/src/core/event_manager.ts index 12ccbfd889b..7b07b66ca92 100644 --- a/packages/firestore/src/core/event_manager.ts +++ b/packages/firestore/src/core/event_manager.ts @@ -23,6 +23,7 @@ import { SyncEngine, SyncEngineListener } from './sync_engine'; import { OnlineState } from './types'; import { ChangeType, DocumentViewChange, ViewSnapshot } from './view_snapshot'; import { wrapInUserErrorIfRecoverable } from '../util/async_queue'; +import { SnapshotVersion } from './snapshot_version'; /** * Holds the listeners and the last received ViewSnapshot for a query being @@ -59,7 +60,10 @@ export class EventManager implements SyncEngineListener { this.syncEngine.subscribe(this); } - async listen(listener: QueryListener): Promise { + async listen( + listener: QueryListener, + readFrom?: SnapshotVersion + ): Promise { const query = listener.query; let firstListen = false; @@ -71,7 +75,7 @@ export class EventManager implements SyncEngineListener { if (firstListen) { try { - queryInfo.viewSnap = await this.syncEngine.listen(query); + queryInfo.viewSnap = await this.syncEngine.listen(query, readFrom); } catch (e) { const firestoreError = wrapInUserErrorIfRecoverable( e, @@ -195,6 +199,16 @@ export interface ListenOptions { * offline. */ readonly waitForSyncWhenOnline?: boolean; + + /** + * Tells the backend whether the client already has the query results up to + * a point in time, and the backend will only send deltas from that point + * on if applicable (for example, backend might already lose track of this + * particular query, and has to restart and send everything anyways). + * + * When not set, backend simply assumes the client does not have anything. + */ + readFrom?: SnapshotVersion | undefined; } /** diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index c69713925fe..9426de9f3a5 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -50,7 +50,7 @@ import { MemoryComponentProvider } from './component_provider'; import { BundleReader } from '../util/bundle_reader'; -import { LoadBundleTaskImpl } from './bundle'; +import { LoadBundleTaskImpl, NamedQuery } from './bundle'; const LOG_TAG = 'FirestoreClient'; const MAX_CONCURRENT_LIMBO_RESOLUTIONS = 100; @@ -396,7 +396,9 @@ export class FirestoreClient { ): QueryListener { this.verifyNotTerminated(); const listener = new QueryListener(query, observer, options); - this.asyncQueue.enqueueAndForget(() => this.eventMgr.listen(listener)); + this.asyncQueue.enqueueAndForget(() => + this.eventMgr.listen(listener, options.readFrom) + ); return listener; } @@ -544,4 +546,9 @@ export class FirestoreClient { return task; } + + getNamedQuery(queryName: string): Promise { + this.verifyNotTerminated(); + return this.localStore.getNamedQuery(queryName); + } } diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index d9096523661..60ed9248e7b 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -216,7 +216,10 @@ export class SyncEngine implements RemoteSyncer { * server. All the subsequent view snapshots or errors are sent to the * subscribed handlers. Returns the initial snapshot. */ - async listen(query: Query): Promise { + async listen( + query: Query, + readFrom?: SnapshotVersion + ): Promise { this.assertSubscribed('listen()'); let targetId; @@ -234,7 +237,10 @@ export class SyncEngine implements RemoteSyncer { this.sharedClientState.addLocalQueryTarget(targetId); viewSnapshot = queryView.view.computeInitialSnapshot(); } else { - const targetData = await this.localStore.allocateTarget(query.toTarget()); + const targetData = await this.localStore.allocateTarget( + query.toTarget(), + readFrom + ); const status = this.sharedClientState.addLocalQueryTarget( targetData.targetId diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 6728bc49ae1..ce89ad8166d 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -960,7 +960,10 @@ export class LocalStore { * Allocating an already allocated `Target` will return the existing `TargetData` * for that `Target`. */ - allocateTarget(target: Target): Promise { + allocateTarget( + target: Target, + readFrom?: SnapshotVersion + ): Promise { return this.persistence .runTransaction('Allocate target', 'readwrite', txn => { let targetData: TargetData; @@ -979,7 +982,8 @@ export class LocalStore { target, targetId, TargetPurpose.Listen, - txn.currentSequenceNumber + txn.currentSequenceNumber, + !!readFrom ? readFrom! : SnapshotVersion.min() ); return this.targetCache .addTargetData(txn, targetData) diff --git a/packages/firestore/src/protos/firestore_proto_api.d.ts b/packages/firestore/src/protos/firestore_proto_api.d.ts index a85bc282bca..84c375b79dd 100644 --- a/packages/firestore/src/protos/firestore_proto_api.d.ts +++ b/packages/firestore/src/protos/firestore_proto_api.d.ts @@ -345,7 +345,7 @@ export declare namespace firestoreV1ApiClientInterfaces { query?: QueryTarget; documents?: DocumentsTarget; resumeToken?: string | Uint8Array; - readTime?: string; + readTime?: Timestamp; targetId?: number; once?: boolean; } diff --git a/packages/firestore/src/remote/serializer.ts b/packages/firestore/src/remote/serializer.ts index db1ff3f3c39..b7730b67353 100644 --- a/packages/firestore/src/remote/serializer.ts +++ b/packages/firestore/src/remote/serializer.ts @@ -887,6 +887,10 @@ export class JsonProtoSerializer { if (targetData.resumeToken.approximateByteSize() > 0) { result.resumeToken = this.toBytes(targetData.resumeToken); + } else if (targetData.snapshotVersion.compareTo(SnapshotVersion.min())) { + result.readTime = this.toTimestamp( + targetData.snapshotVersion.toTimestamp() + ); } return result; From cd884144ad7333b1f47aef3a120be2b97bac60ac Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Sun, 14 Jun 2020 23:32:45 -0400 Subject: [PATCH 31/43] Implements bundled query loading and resuming. --- packages/firestore-types/index.d.ts | 2 + .../test/integration/api/bundle.test.ts | 78 +++++++--- .../test/unit/specs/bundle_spec.test.ts | 135 ++++++++++++++++-- .../firestore/test/unit/specs/spec_builder.ts | 23 ++- .../test/unit/specs/spec_test_runner.ts | 47 ++++-- packages/firestore/test/util/bundle_data.ts | 19 ++- 6 files changed, 248 insertions(+), 56 deletions(-) diff --git a/packages/firestore-types/index.d.ts b/packages/firestore-types/index.d.ts index 7c543b6a9c6..ef56c1819ba 100644 --- a/packages/firestore-types/index.d.ts +++ b/packages/firestore-types/index.d.ts @@ -89,6 +89,8 @@ export class FirebaseFirestore { bundleData: ArrayBuffer | ReadableStream | string ): LoadBundleTask; + namedQuery(name: string): Promise; + INTERNAL: { delete: () => Promise }; } diff --git a/packages/firestore/test/integration/api/bundle.test.ts b/packages/firestore/test/integration/api/bundle.test.ts index c5097fd75dd..9ddb8ae426a 100644 --- a/packages/firestore/test/integration/api/bundle.test.ts +++ b/packages/firestore/test/integration/api/bundle.test.ts @@ -27,6 +27,7 @@ import { TestBundleBuilder } from '../../util/bundle_data'; import { DatabaseId } from '../../../src/core/database_info'; import { key } from '../../util/helpers'; import { EventsAccumulator } from '../util/events_accumulator'; +import { collectionReference } from '../../util/api_helpers'; function verifySuccessProgress(p: firestore.LoadBundleTaskProgress): void { expect(p.taskState).to.equal('Success'); @@ -51,7 +52,7 @@ apiDescribe('Bundles', (persistence: boolean) => { b: { k: { stringValue: 'b' }, bar: { integerValue: 2 } } }; - function bundleWithTestDocs( + function bundleWithTestDocsAndQueries( db: firestore.FirebaseFirestore ): TestBundleBuilder { const a = key('coll-1/a'); @@ -60,6 +61,25 @@ apiDescribe('Bundles', (persistence: boolean) => { // eslint-disable-next-line @typescript-eslint/no-explicit-any (db as any)._databaseId as DatabaseId ); + + builder.addNamedQuery( + 'limit', + { seconds: 1000, nanos: 9999 }, + (collectionReference('coll-1') + .orderBy('bar', 'desc') + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .limit(1) as any)._query + ); + builder.addNamedQuery( + 'limit-to-last', + { seconds: 1000, nanos: 9999 }, + (collectionReference('coll-1') + .orderBy('bar', 'desc') + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .limit(1) as any)._query, + 'LAST' + ); + builder.addDocumentMetadata(a, { seconds: 1000, nanos: 9999 }, true); builder.addDocument( a, @@ -80,7 +100,7 @@ apiDescribe('Bundles', (persistence: boolean) => { it('load with documents only with on progress and promise interface.', () => { return withTestDb(persistence, async db => { - const builder = bundleWithTestDocs(db); + const builder = bundleWithTestDocsAndQueries(db); const progresses: firestore.LoadBundleTaskProgress[] = []; let completeProgress: firestore.LoadBundleTaskProgress, @@ -110,24 +130,35 @@ apiDescribe('Bundles', (persistence: boolean) => { verifySuccessProgress(completeProgress!); verifySuccessProgress(fulfillProgress!); - expect(progresses.length).to.equal(3); + // 2 named queries + 2 documents + initial progress. + expect(progresses.length).to.equal(5); verifyInProgress(progresses[0], 0); - verifyInProgress(progresses[1], 1); - verifyInProgress(progresses[2], 2); + verifyInProgress(progresses[1], 0); + verifyInProgress(progresses[2], 0); + verifyInProgress(progresses[3], 1); + verifyInProgress(progresses[4], 2); // Read from cache. These documents do not exist in backend, so they can // only be read from cache. - const snap = await db.collection('coll-1').get({ source: 'cache' }); + let snap = await db.collection('coll-1').get({ source: 'cache' }); expect(toDataArray(snap)).to.deep.equal([ { k: 'a', bar: 1 }, { k: 'b', bar: 2 } ]); + + snap = await (await db.namedQuery('limit'))!.get({ source: 'cache' }); + expect(toDataArray(snap)).to.deep.equal([{ k: 'b', bar: 2 }]); + + snap = await (await db.namedQuery('limit-to-last'))!.get({ + source: 'cache' + }); + expect(toDataArray(snap)).to.deep.equal([{ k: 'a', bar: 1 }]); }); }); - it('load with documents with promise interface.', () => { + it('load with documents and queries with promise interface.', () => { return withTestDb(persistence, async db => { - const builder = bundleWithTestDocs(db); + const builder = bundleWithTestDocsAndQueries(db); let fulfillProgress: firestore.LoadBundleTaskProgress; await db @@ -145,17 +176,16 @@ apiDescribe('Bundles', (persistence: boolean) => { // Read from cache. These documents do not exist in backend, so they can // only be read from cache. - const snap = await db.collection('coll-1').get({ source: 'cache' }); - expect(toDataArray(snap)).to.deep.equal([ - { k: 'a', bar: 1 }, - { k: 'b', bar: 2 } - ]); + const snap = await (await db.namedQuery('limit-to-last'))!.get({ + source: 'cache' + }); + expect(toDataArray(snap)).to.deep.equal([{ k: 'a', bar: 1 }]); }); }); it('load for a second time skips.', () => { return withTestDb(persistence, async db => { - const builder = bundleWithTestDocs(db); + const builder = bundleWithTestDocsAndQueries(db); await db.loadBundle( builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) @@ -202,7 +232,7 @@ apiDescribe('Bundles', (persistence: boolean) => { db.collection('coll-1').onSnapshot(accumulator.storeEvent); await accumulator.awaitEvent(); - const builder = bundleWithTestDocs(db); + const builder = bundleWithTestDocsAndQueries(db); const progress = await db.loadBundle( // Testing passing non-string bundles. encoder.encode( @@ -216,17 +246,19 @@ apiDescribe('Bundles', (persistence: boolean) => { // cache can only be tested in spec tests. await accumulator.assertNoAdditionalEvents(); - const snap = await db.collection('coll-1').get(); - expect(toDataArray(snap)).to.deep.equal([ - { k: 'a', bar: 0 }, - { k: 'b', bar: 0 } - ]); + let snap = await (await db.namedQuery('limit'))!.get({ source: 'cache' }); + expect(toDataArray(snap)).to.deep.equal([{ k: 'b', bar: 0 }]); + + snap = await (await db.namedQuery('limit-to-last'))!.get({ + source: 'cache' + }); + expect(toDataArray(snap)).to.deep.equal([{ k: 'a', bar: 0 }]); }); }); - it('load with documents from other projects fails.', () => { + it('load bunldes from other projects fails.', () => { return withTestDb(persistence, async db => { - let builder = bundleWithTestDocs(db); + let builder = bundleWithTestDocsAndQueries(db); return withAlternateTestDb(persistence, async otherDb => { // eslint-disable-next-line @typescript-eslint/no-floating-promises expect( @@ -236,7 +268,7 @@ apiDescribe('Bundles', (persistence: boolean) => { ).to.be.rejectedWith('Tried to deserialize key from different project'); // Verify otherDb still functions, despite loaded a problematic bundle. - builder = bundleWithTestDocs(otherDb); + builder = bundleWithTestDocsAndQueries(otherDb); const progress = await otherDb.loadBundle( builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ); diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts index 428f7696519..38625c1735b 100644 --- a/packages/firestore/test/unit/specs/bundle_spec.test.ts +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -16,7 +16,13 @@ */ import { Query } from '../../../src/core/query'; -import { doc, path, TestSnapshotVersion, version } from '../../util/helpers'; +import { + doc, + filter, + path, + TestSnapshotVersion, + version +} from '../../util/helpers'; import { describeSpec, specTest } from './describe_spec'; import { client, spec } from './spec_builder'; @@ -28,6 +34,8 @@ import { import { DocumentKey } from '../../../src/model/document_key'; import * as api from '../../../src/protos/firestore_proto_api'; import { Value } from '../../../src/protos/firestore_proto_api'; +import { BundledQuery } from '../../../src/protos/firestore_bundle_proto'; +import LimitType = BundledQuery.LimitType; interface TestBundleDocument { key: DocumentKey; @@ -36,8 +44,29 @@ interface TestBundleDocument { updateTime?: TestSnapshotVersion; content?: api.ApiClientObjectMap; } -function bundleWithDocument(testDoc: TestBundleDocument): string { + +interface TestBundledQuery { + name: string; + readTime: TestSnapshotVersion; + query: Query; + limitType?: LimitType; +} + +function bundleWithDocumentAndQuery( + testDoc: TestBundleDocument, + testQuery?: TestBundledQuery +): string { const builder = new TestBundleBuilder(TEST_DATABASE_ID); + + if (testQuery) { + builder.addNamedQuery( + testQuery.name, + JSON_SERIALIZER.toVersion(version(testQuery.readTime)), + testQuery.query, + testQuery.limitType + ); + } + builder.addDocumentMetadata( testDoc.key, JSON_SERIALIZER.toVersion(version(testDoc.readTime)), @@ -63,7 +92,7 @@ describeSpec('Bundles:', [], () => { const docA = doc('collection/a', 1000, { key: 'a' }); const docAChanged = doc('collection/a', 2999, { key: 'b' }); - const bundleString = bundleWithDocument({ + const bundleString = bundleWithDocumentAndQuery({ key: docA.key, readTime: 3000, createTime: 1999, @@ -83,7 +112,7 @@ describeSpec('Bundles:', [], () => { const query1 = Query.atPath(path('collection')); const docA = doc('collection/a', 1000, { key: 'a' }); - const bundleString = bundleWithDocument({ + const bundleString = bundleWithDocumentAndQuery({ key: docA.key, readTime: 3000 }); @@ -100,7 +129,7 @@ describeSpec('Bundles:', [], () => { const query1 = Query.atPath(path('collection')); const docA = doc('collection/a', 1000, { key: 'a' }); - const bundleString = bundleWithDocument({ + const bundleString = bundleWithDocumentAndQuery({ key: docA.key, readTime: 999 }); @@ -122,7 +151,7 @@ describeSpec('Bundles:', [], () => { const query = Query.atPath(path('collection')); const docA = doc('collection/a', 250, { key: 'a' }); - const bundleString1 = bundleWithDocument({ + const bundleString1 = bundleWithDocumentAndQuery({ key: docA.key, readTime: 500, createTime: 250, @@ -130,7 +159,7 @@ describeSpec('Bundles:', [], () => { content: { key: { stringValue: 'b' } } }); - const bundleString2 = bundleWithDocument({ + const bundleString2 = bundleWithDocumentAndQuery({ key: docA.key, readTime: 1001, createTime: 250, @@ -178,7 +207,7 @@ describeSpec('Bundles:', [], () => { const query = Query.atPath(path('collection')); const docA = doc('collection/a', 250, { key: 'a' }); - const bundleString1 = bundleWithDocument({ + const bundleString1 = bundleWithDocumentAndQuery({ key: docA.key, readTime: 500, createTime: 250, @@ -186,7 +215,7 @@ describeSpec('Bundles:', [], () => { content: { key: { stringValue: 'b' } } }); - const bundleString2 = bundleWithDocument({ + const bundleString2 = bundleWithDocumentAndQuery({ key: docA.key, readTime: 1001, createTime: 250, @@ -225,7 +254,7 @@ describeSpec('Bundles:', [], () => { specTest('Newer docs from bundles might lead to limbo doc.', [], () => { const query = Query.atPath(path('collection')); const docA = doc('collection/a', 1000, { key: 'a' }); - const bundleString1 = bundleWithDocument({ + const bundleString1 = bundleWithDocumentAndQuery({ key: docA.key, readTime: 500, createTime: 250, @@ -250,13 +279,95 @@ describeSpec('Bundles:', [], () => { ); }); + specTest('Bundles query can be loaded and resumed.', [], () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 100, { key: 'a' }); + const bundleString1 = bundleWithDocumentAndQuery( + { + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } + }, + { name: 'bundled-query', readTime: 400, query } + ); + + return spec() + .loadBundle(bundleString1) + .userListensToNamedQuery('bundled-query', query, 400) + .expectEvents(query, { + added: [doc('collection/a', 500, { key: 'b' })], + fromCache: true + }); + }); + + specTest( + 'Bundles query can be loaded and resumed from different tabs', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const query1 = Query.atPath(path('collection')).addFilter( + filter('key', '==', 'c') + ); + const docA = doc('collection/a', 100, { key: 'a' }); + const bundleString1 = bundleWithDocumentAndQuery( + { + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } + }, + { name: 'bundled-query', readTime: 400, query } + ); + + const bundleString2 = bundleWithDocumentAndQuery( + { + key: docA.key, + readTime: 600, + createTime: 250, + updateTime: 550, + content: { key: { stringValue: 'c' } } + }, + { name: 'bundled-query', readTime: 560, query: query1 } + ); + + return ( + client(0) + .loadBundle(bundleString1) + // Read named query from loaded bundle by primary. + .client(1) + .userListensToNamedQuery('bundled-query', query, 400) + .expectEvents(query, { + added: [doc('collection/a', 500, { key: 'b' })], + fromCache: true + }) + // Loads a newer bundle. + .loadBundle(bundleString2) + .expectEvents(query, { + modified: [doc('collection/a', 550, { key: 'c' })], + fromCache: true + }) + .userUnlistens(query) + // Read named query from loaded bundle by secondary. + .client(0) + .userListensToNamedQuery('bundled-query', query1, 560) + .expectEvents(query1, { + added: [doc('collection/a', 550, { key: 'c' })], + fromCache: true + }) + ); + } + ); + specTest( 'Load and observe from secondary and observe from others.', ['multi-client'], () => { const query = Query.atPath(path('collection')); const docA = doc('collection/a', 250, { key: 'a' }); - const bundleString1 = bundleWithDocument({ + const bundleString1 = bundleWithDocumentAndQuery({ key: docA.key, readTime: 500, createTime: 250, @@ -301,7 +412,7 @@ describeSpec('Bundles:', [], () => { () => { const query = Query.atPath(path('collection')); const docA = doc('collection/a', 250, { key: 'a' }); - const bundleString1 = bundleWithDocument({ + const bundleString1 = bundleWithDocumentAndQuery({ key: docA.key, readTime: 500, createTime: 250, diff --git a/packages/firestore/test/unit/specs/spec_builder.ts b/packages/firestore/test/unit/specs/spec_builder.ts index cf728c675a0..264e149241a 100644 --- a/packages/firestore/test/unit/specs/spec_builder.ts +++ b/packages/firestore/test/unit/specs/spec_builder.ts @@ -66,7 +66,7 @@ export interface LimboMap { export interface ActiveTargetSpec { queries: SpecQuery[]; - resumeToken: string; + resumeToken: string | TestSnapshotVersion; } export interface ActiveTargetMap { @@ -237,7 +237,7 @@ export class SpecBuilder { return this; } - userListens(query: Query, resumeToken?: string): this { + userListens(query: Query, resumeFrom?: string | TestSnapshotVersion): this { this.nextStep(); const target = query.toTarget(); @@ -246,7 +246,7 @@ export class SpecBuilder { if (this.injectFailures) { // Return a `userListens()` step but don't advance the target IDs. this.currentStep = { - userListen: [targetId, SpecBuilder.queryToSpec(query)] + userListen: { targetId, query: SpecBuilder.queryToSpec(query) } }; } else { if (this.queryMapping.has(target)) { @@ -256,9 +256,9 @@ export class SpecBuilder { } this.queryMapping.set(target, targetId); - this.addQueryToActiveTargets(targetId, query, resumeToken); + this.addQueryToActiveTargets(targetId, query, resumeFrom); this.currentStep = { - userListen: [targetId, SpecBuilder.queryToSpec(query)], + userListen: { targetId, query: SpecBuilder.queryToSpec(query) }, expectedState: { activeTargets: { ...this.activeTargets } } }; } @@ -353,6 +353,17 @@ export class SpecBuilder { return this; } + userListensToNamedQuery( + name: string, + query: Query, + readFrom: TestSnapshotVersion + ): this { + // Note we do not call this.nextStep() here because we reuse userListens(). + this.userListens(query, readFrom); + this.currentStep!.userListen!.fromName = name; + return this; + } + // PORTING NOTE: Only used by web multi-tab tests. becomeHidden(): this { this.nextStep(); @@ -1013,7 +1024,7 @@ export class SpecBuilder { private addQueryToActiveTargets( targetId: number, query: Query, - resumeToken?: string + resumeToken?: string | TestSnapshotVersion ): void { if (this.activeTargets[targetId]) { const activeQueries = this.activeTargets[targetId].queries; diff --git a/packages/firestore/test/unit/specs/spec_test_runner.ts b/packages/firestore/test/unit/specs/spec_test_runner.ts index 1ad9c3cf685..a74bf643dac 100644 --- a/packages/firestore/test/unit/specs/spec_test_runner.ts +++ b/packages/firestore/test/unit/specs/spec_test_runner.ts @@ -356,8 +356,16 @@ abstract class TestRunner { private async doListen(listenSpec: SpecUserListen): Promise { let targetFailed = false; - const querySpec = listenSpec[1]; + const querySpec = listenSpec.query; const query = parseQuery(querySpec); + let readFrom: SnapshotVersion; + if (listenSpec.fromName) { + const savedQuery = await this.localStore.getNamedQuery( + listenSpec.fromName + ); + expect(query.isEqual(savedQuery!.query)).to.be.true; + readFrom = savedQuery!.readTime; + } const aggregator = new EventAggregator(query, e => { if (e.error) { targetFailed = true; @@ -372,7 +380,9 @@ abstract class TestRunner { const queryListener = new QueryListener(query, aggregator, options); this.queryListeners.set(query, queryListener); - await this.queue.enqueue(() => this.eventManager.listen(queryListener)); + await this.queue.enqueue(() => + this.eventManager.listen(queryListener, readFrom) + ); if (targetFailed) { expect(this.persistence.injectFailures).contains('Allocate target'); @@ -955,17 +965,24 @@ abstract class TestRunner { // TODO(mcg): populate the purpose of the target once it's possible to // encode that in the spec tests. For now, hard-code that it's a listen // despite the fact that it's not always the right value. - const expectedTarget = this.serializer.toTarget( - new TargetData( - parseQuery(expected.queries[0]).toTarget(), - targetId, - TargetPurpose.Listen, - ARBITRARY_SEQUENCE_NUMBER, - SnapshotVersion.min(), - SnapshotVersion.min(), - byteStringFromString(expected.resumeToken) - ) + let targetData = new TargetData( + parseQuery(expected.queries[0]).toTarget(), + targetId, + TargetPurpose.Listen, + ARBITRARY_SEQUENCE_NUMBER ); + if (typeof expected.resumeToken === 'string') { + targetData = targetData.withResumeToken( + byteStringFromString(expected.resumeToken), + SnapshotVersion.min() + ); + } else { + targetData = targetData.withResumeToken( + byteStringFromString(''), + version(expected.resumeToken) + ); + } + const expectedTarget = this.serializer.toTarget(targetData); expect(actualTarget.query).to.deep.equal(expectedTarget.query); expect(actualTarget.targetId).to.equal(expectedTarget.targetId); expect(actualTarget.readTime).to.equal(expectedTarget.readTime); @@ -1343,7 +1360,11 @@ export interface SpecStep { } /** [, ] */ -export type SpecUserListen = [TargetId, string | SpecQuery]; +export interface SpecUserListen { + targetId: TargetId; + query: string | SpecQuery; + fromName?: string; +} /** [, ] */ export type SpecUserUnlisten = [TargetId, string | SpecQuery]; diff --git a/packages/firestore/test/util/bundle_data.ts b/packages/firestore/test/util/bundle_data.ts index 125acbb077b..c179b02c3b2 100644 --- a/packages/firestore/test/util/bundle_data.ts +++ b/packages/firestore/test/util/bundle_data.ts @@ -24,6 +24,7 @@ import { Value } from '../../src/protos/firestore_proto_api'; import { JsonProtoSerializer } from '../../src/remote/serializer'; import { PlatformSupport } from '../../src/platform/platform'; import { DocumentKey } from '../../src/model/document_key'; +import { Query } from '../../src/core/query'; function lengthPrefixedString(o: {}): string { const str = JSON.stringify(o); @@ -68,14 +69,28 @@ export class TestBundleBuilder { }); return this; } + addNamedQuery( name: string, readTime: api.Timestamp, - bundledQuery: BundledQuery + query: Query, + limitType?: BundledQuery.LimitType ): TestBundleBuilder { - this.elements.push({ namedQuery: { name, readTime, bundledQuery } }); + const queryTarget = this.serializer.toQueryTarget(query.toTarget()); + this.elements.push({ + namedQuery: { + name, + readTime, + bundledQuery: { + parent: queryTarget.parent, + structuredQuery: queryTarget.structuredQuery, + limitType + } + } + }); return this; } + getMetadataElement( id: string, createTime: api.Timestamp, From d738d9d38d6f5428d2ce23b0b3cb9652715e2036 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Sat, 11 Jul 2020 21:18:41 -0400 Subject: [PATCH 32/43] Undo rename --- packages/firestore/src/core/snapshot_version.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/firestore/src/core/snapshot_version.ts b/packages/firestore/src/core/snapshot_version.ts index 12566c40543..b73ea960c9e 100644 --- a/packages/firestore/src/core/snapshot_version.ts +++ b/packages/firestore/src/core/snapshot_version.ts @@ -30,27 +30,27 @@ export class SnapshotVersion { return new SnapshotVersion(new Timestamp(0, 0)); } - private constructor(private _timestamp: Timestamp) {} + private constructor(private timestamp: Timestamp) {} compareTo(other: SnapshotVersion): number { - return this._timestamp._compareTo(other._timestamp); + return this.timestamp._compareTo(other.timestamp); } isEqual(other: SnapshotVersion): boolean { - return this._timestamp.isEqual(other._timestamp); + return this.timestamp.isEqual(other.timestamp); } /** Returns a number representation of the version for use in spec tests. */ toMicroseconds(): number { // Convert to microseconds. - return this._timestamp.seconds * 1e6 + this._timestamp.nanoseconds / 1000; + return this.timestamp.seconds * 1e6 + this.timestamp.nanoseconds / 1000; } toString(): string { - return 'SnapshotVersion(' + this._timestamp.toString() + ')'; + return 'SnapshotVersion(' + this.timestamp.toString() + ')'; } toTimestamp(): Timestamp { - return this._timestamp; + return this.timestamp; } } From 23ea4d843eb1b6989fd387d4be08ffab9f8bbe77 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Mon, 27 Jul 2020 16:07:56 -0400 Subject: [PATCH 33/43] Make loadBundle work with exp build --- packages/firestore/exp-types/index.d.ts | 5 +++++ packages/firestore/exp/src/api/database.ts | 15 ++++++++++----- packages/firestore/exp/test/shim.ts | 3 ++- packages/firestore/src/api/database.ts | 5 ++++- packages/firestore/src/core/firestore_client.ts | 13 +++++-------- 5 files changed, 26 insertions(+), 15 deletions(-) diff --git a/packages/firestore/exp-types/index.d.ts b/packages/firestore/exp-types/index.d.ts index f27b50dea99..170a553a506 100644 --- a/packages/firestore/exp-types/index.d.ts +++ b/packages/firestore/exp-types/index.d.ts @@ -536,6 +536,11 @@ export interface LoadBundleTaskProgress { export type TaskState = 'Error' | 'Running' | 'Success'; +export function loadBundle( + firestore: FirebaseFirestore, + bundleData: ArrayBuffer | ReadableStream | string +): LoadBundleTask; + export type FirestoreErrorCode = | 'cancelled' | 'unknown' diff --git a/packages/firestore/exp/src/api/database.ts b/packages/firestore/exp/src/api/database.ts index 13e4be085dc..d0004141c16 100644 --- a/packages/firestore/exp/src/api/database.ts +++ b/packages/firestore/exp/src/api/database.ts @@ -50,6 +50,7 @@ import { indexedDbStoragePrefix, indexedDbClearPersistence } from '../../../src/local/indexeddb_persistence'; +import { LoadBundleTask } from '../../../src/api/bundle'; /** * The root reference to the Firestore database and the entry point for the @@ -299,9 +300,13 @@ export function terminate( export function loadBundle( firestore: firestore.FirebaseFirestore, bundleData: ArrayBuffer | ReadableStream | string -): firestore.LoadBundleTask | null { - return null; - // const firestoreImpl = cast(firestore, Firestore); - // return firestoreImpl._getFirestoreClient() - // .then(firestoreClient => firestoreClient.loadBundle(bundleData)); +): LoadBundleTask { + const firestoreImpl = cast(firestore, Firestore); + const resultTask = new LoadBundleTask(); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + firestoreImpl._getFirestoreClient().then(firestoreClient => { + firestoreClient.loadBundle(bundleData, resultTask); + }); + + return resultTask; } diff --git a/packages/firestore/exp/test/shim.ts b/packages/firestore/exp/test/shim.ts index 618c176cedb..bf53a1a0be0 100644 --- a/packages/firestore/exp/test/shim.ts +++ b/packages/firestore/exp/test/shim.ts @@ -72,6 +72,7 @@ import { import { UntypedFirestoreDataConverter } from '../../src/api/user_data_reader'; import { isPartialObserver, PartialObserver } from '../../src/api/observer'; import { isPlainObject } from '../../src/util/input_validation'; +import { LoadBundleTask } from '../../exp-types'; export { GeoPoint, Blob, Timestamp } from '../index'; @@ -168,7 +169,7 @@ export class FirebaseFirestore implements legacy.FirebaseFirestore { loadBundle( bundleData: ArrayBuffer | ReadableStream | string - ): legacy.LoadBundleTask { + ): LoadBundleTask { return loadBundle(this._delegate, bundleData)!; } diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index 9c496ee23e6..e196a56e5b7 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -112,6 +112,7 @@ import { import { UserDataWriter } from './user_data_writer'; import { FirebaseAuthInternalName } from '@firebase/auth-interop-types'; import { Provider } from '@firebase/component'; +import { LoadBundleTask } from './bundle'; // settings() defaults: const DEFAULT_HOST = 'firestore.googleapis.com'; @@ -498,7 +499,9 @@ export class Firestore implements firestore.FirebaseFirestore, FirebaseService { bundleData: ArrayBuffer | ReadableStream | string ): firestore.LoadBundleTask { this.ensureClientConfigured(); - return this._firestoreClient!.loadBundle(bundleData); + const resultTask = new LoadBundleTask(); + this._firestoreClient!.loadBundle(bundleData, resultTask); + return resultTask; } ensureClientConfigured(): FirestoreClient { diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index d597e459a45..011c92500b0 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -15,7 +15,6 @@ * limitations under the License. */ -import * as firestore from '@firebase/firestore-types'; import { CredentialsProvider } from '../api/credentials'; import { User } from '../auth/user'; import { LocalStore } from '../local/local_store'; @@ -515,8 +514,9 @@ export class FirestoreClient { } loadBundle( - data: ReadableStream | ArrayBuffer | string - ): firestore.LoadBundleTask { + data: ReadableStream | ArrayBuffer | string, + resultTask: LoadBundleTask + ): void { this.verifyNotTerminated(); let content: ReadableStream | ArrayBuffer; @@ -526,14 +526,11 @@ export class FirestoreClient { content = data; } const reader = new BundleReader(toByteStreamReader(content)); - const task = new LoadBundleTask(); this.asyncQueue.enqueueAndForget(async () => { - loadBundle(this.syncEngine, reader, task); - return task.catch(e => { + loadBundle(this.syncEngine, reader, resultTask); + return resultTask.catch(e => { logWarn(LOG_TAG, `Loading bundle failed with ${e}`); }); }); - - return task; } } From 27e0fd851f789bd367898a3f0964fc30c3c1da6a Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Mon, 27 Jul 2020 16:32:33 -0400 Subject: [PATCH 34/43] Add TODO --- packages/firestore/exp/src/api/database.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/firestore/exp/src/api/database.ts b/packages/firestore/exp/src/api/database.ts index f047e647118..aa87460bbb4 100644 --- a/packages/firestore/exp/src/api/database.ts +++ b/packages/firestore/exp/src/api/database.ts @@ -44,7 +44,7 @@ import { cast } from '../../../lite/src/api/util'; import { Code, FirestoreError } from '../../../src/util/error'; import { Deferred } from '../../../src/util/promise'; import { LruParams } from '../../../src/local/lru_garbage_collector'; -import { CACHE_SIZE_UNLIMITED } from '../../../src/api/database'; +import { CACHE_SIZE_UNLIMITED, Query } from '../../../src/api/database'; import { DatabaseId, DatabaseInfo } from '../../../src/core/database_info'; import { indexedDbStoragePrefix, @@ -316,6 +316,8 @@ export async function namedQuery( if (!namedQuery) { return null; } + + // TODO(wuandy): make this work with exp build. return null; - // return new firestore.Query(namedQuery.query, cast(firestoreImpl, LegacyFirestore), null, namedQuery.readTime); + // return new Query(namedQuery.query, firestoreImpl, null, namedQuery.readTime); } From a3d0bb0c09e359aa6e3c199d358b81ae29656f2a Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Mon, 27 Jul 2020 16:44:10 -0400 Subject: [PATCH 35/43] Undo some merges --- .../firestore/src/platform/browser/dom.ts | 14 ---------- packages/firestore/src/platform/dom.ts | 27 ------------------- packages/firestore/src/platform/node/dom.ts | 16 ----------- packages/firestore/src/platform/rn/dom.ts | 7 +---- 4 files changed, 1 insertion(+), 63 deletions(-) diff --git a/packages/firestore/src/platform/browser/dom.ts b/packages/firestore/src/platform/browser/dom.ts index 3b5d10fb63c..5c42994e19e 100644 --- a/packages/firestore/src/platform/browser/dom.ts +++ b/packages/firestore/src/platform/browser/dom.ts @@ -28,17 +28,3 @@ export function getDocument(): Document | null { // eslint-disable-next-line no-restricted-globals return typeof document !== 'undefined' ? document : null; } - -/** - * An instance of the Platform's 'TextEncoder' implementation. - */ -export function newTextEncoder(): TextEncoder { - return new TextEncoder(); -} - -/** - * An instance of the Platform's 'TextDecoder' implementation. - */ -export function newTextDecoder(): TextDecoder { - return new TextDecoder('utf-8'); -} diff --git a/packages/firestore/src/platform/dom.ts b/packages/firestore/src/platform/dom.ts index 0ed9627be17..33090224beb 100644 --- a/packages/firestore/src/platform/dom.ts +++ b/packages/firestore/src/platform/dom.ts @@ -41,30 +41,3 @@ export function getDocument(): Document | null { return browser.getDocument(); } } - -/** - * An instance of the Platform's 'TextEncoder' implementation or null - * if not available. - */ -export function newTextEncoder(): TextEncoder { - if (isNode()) { - return node.newTextEncoder(); - } else if (isReactNative()) { - return rn.newTextEncoder(); - } else { - return browser.newTextEncoder(); - } -} - -/** - * An instance of the Platform's 'TextDecoder' implementation. - */ -export function newTextDecoder(): TextDecoder { - if (isNode()) { - return node.newTextDecoder() as TextDecoder; - } else if (isReactNative()) { - return rn.newTextDecoder(); - } else { - return browser.newTextDecoder(); - } -} diff --git a/packages/firestore/src/platform/node/dom.ts b/packages/firestore/src/platform/node/dom.ts index c55f1e933d9..a1c00e98bfd 100644 --- a/packages/firestore/src/platform/node/dom.ts +++ b/packages/firestore/src/platform/node/dom.ts @@ -15,8 +15,6 @@ * limitations under the License. */ -import { TextEncoder, TextDecoder } from 'util'; - /** The Platform's 'window' implementation or null if not available. */ export function getWindow(): Window | null { if (process.env.USE_MOCK_PERSISTENCE === 'YES') { @@ -31,17 +29,3 @@ export function getWindow(): Window | null { export function getDocument(): Document | null { return null; } - -/** - * An instance of the Platform's 'TextEncoder' implementation. - */ -export function newTextEncoder(): TextEncoder { - return new TextEncoder(); -} - -/** - * An instance of the Platform's 'TextDecoder' implementation. - */ -export function newTextDecoder(): TextDecoder { - return new TextDecoder('utf-8'); -} diff --git a/packages/firestore/src/platform/rn/dom.ts b/packages/firestore/src/platform/rn/dom.ts index 657ff4135ca..87e6870db9e 100644 --- a/packages/firestore/src/platform/rn/dom.ts +++ b/packages/firestore/src/platform/rn/dom.ts @@ -15,9 +15,4 @@ * limitations under the License. */ -export { - getWindow, - getDocument, - newTextEncoder, - newTextDecoder -} from '../browser/dom'; +export { getWindow, getDocument } from '../browser/dom'; From 8baf8fc3c3457293bc60a652868cc55074a655bb Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 28 Jul 2020 13:44:45 -0400 Subject: [PATCH 36/43] Use legacy.LoadBundleTask --- packages/firestore/exp/test/shim.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/firestore/exp/test/shim.ts b/packages/firestore/exp/test/shim.ts index bf53a1a0be0..592ffa9c60b 100644 --- a/packages/firestore/exp/test/shim.ts +++ b/packages/firestore/exp/test/shim.ts @@ -169,7 +169,7 @@ export class FirebaseFirestore implements legacy.FirebaseFirestore { loadBundle( bundleData: ArrayBuffer | ReadableStream | string - ): LoadBundleTask { + ): legacy.LoadBundleTask { return loadBundle(this._delegate, bundleData)!; } From f228dfb965bd979e0d3c52eb1ddaef78b77cf6a0 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 28 Jul 2020 17:27:48 -0400 Subject: [PATCH 37/43] Address first round feedbacks --- packages/firestore/exp/test/shim.ts | 4 +++- packages/firestore/src/api/database.ts | 4 ++-- packages/firestore/src/core/sync_engine.ts | 3 +++ packages/firestore/src/local/local_store.ts | 16 ++++++++++++++-- packages/firestore/src/local/target_data.ts | 13 +++++++++++++ 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/packages/firestore/exp/test/shim.ts b/packages/firestore/exp/test/shim.ts index adcdcdcde04..0e58bb7592c 100644 --- a/packages/firestore/exp/test/shim.ts +++ b/packages/firestore/exp/test/shim.ts @@ -175,7 +175,9 @@ export class FirebaseFirestore implements legacy.FirebaseFirestore { } async namedQuery(name: string): Promise { - return namedQuery(this._delegate, name) as Promise; + return namedQuery(this._delegate, name).then(query => { + return query ? new Query(this, query) : null; + }); } INTERNAL = { diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index b906a32546f..59584a6d82b 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -1937,7 +1937,7 @@ export class Query implements firestore.Query { public _query: InternalQuery, readonly firestore: Firestore, protected readonly _converter: firestore.FirestoreDataConverter | null, - private _readFrom?: SnapshotVersion + private _readTime?: SnapshotVersion ) {} where( @@ -2236,7 +2236,7 @@ export class Query implements firestore.Query { firestoreClient, this._query, { - readFrom: this._readFrom, + readFrom: this._readTime, includeMetadataChanges: options.includeMetadataChanges }, observer diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 33e0c4aa199..5656edcc81f 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -174,6 +174,9 @@ export interface SyncEngine extends RemoteSyncer { * Initiates the new listen, resolves promise when listen enqueued to the * server. All the subsequent view snapshots or errors are sent to the * subscribed handlers. Returns the initial snapshot. + * + * @param {SnapshotVersion} readFrom If provided, it tells the backend to + * only return deltas from this snapshot version on. */ listen(query: Query, readFrom?: SnapshotVersion): Promise; diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 4cf7d6e961f..89feefd5fed 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -236,6 +236,10 @@ export interface LocalStore { * they don't get GC'd. A target must be allocated in the local store before * the store can be used to manage its view. * + * @param {SnapshotVersion} readFrom The snapshot version the allocated target + * should resume from, unless the existing `TargetData` has an even newer + * version. + * * Allocating an already allocated `Target` will return the existing `TargetData` * for that `Target`. */ @@ -905,8 +909,16 @@ class LocalStoreImpl implements LocalStore { .next((cached: TargetData | null) => { if (cached) { // This target has been listened to previously, so reuse the - // previous targetID. + // previous targetID if no `readFrom` is provided. + // If `readFrom` is provided, the cached target data needs to be + // compared and updated if `readFrom` is newer. // TODO(mcg): freshen last accessed date? + if (readFrom && cached.snapshotVersion.compareTo(readFrom) < 0) { + targetData = cached.withSnapshotVersion(readFrom); + return this.targetCache + .updateTargetData(txn, targetData) + .next(() => targetData); + } targetData = cached; return PersistencePromise.resolve(targetData); } else { @@ -916,7 +928,7 @@ class LocalStoreImpl implements LocalStore { targetId, TargetPurpose.Listen, txn.currentSequenceNumber, - readFrom || SnapshotVersion.min() + readFrom ); return this.targetCache .addTargetData(txn, targetData) diff --git a/packages/firestore/src/local/target_data.ts b/packages/firestore/src/local/target_data.ts index 95237e574a7..19ffa22a654 100644 --- a/packages/firestore/src/local/target_data.ts +++ b/packages/firestore/src/local/target_data.ts @@ -69,6 +69,19 @@ export class TargetData { readonly resumeToken: ByteString = ByteString.EMPTY_BYTE_STRING ) {} + /** Creates a new target data instance with an updated sequence number. */ + withSnapshotVersion(version: SnapshotVersion): TargetData { + return new TargetData( + this.target, + this.targetId, + this.purpose, + this.sequenceNumber, + version, + this.lastLimboFreeSnapshotVersion, + this.resumeToken + ); + } + /** Creates a new target data instance with an updated sequence number. */ withSequenceNumber(sequenceNumber: number): TargetData { return new TargetData( From 70d350aecb549fdc1289f7c88adf9818e0aa53df Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 31 Jul 2020 12:39:10 -0400 Subject: [PATCH 38/43] Save readtime to target cache and remove readtime from Query --- packages/firestore/src/api/database.ts | 6 +- packages/firestore/src/core/event_manager.ts | 17 +- .../firestore/src/core/firestore_client.ts | 4 +- packages/firestore/src/core/sync_engine.ts | 13 +- .../src/local/indexeddb_bundle_cache.ts | 12 +- .../firestore/src/local/local_serializer.ts | 27 +--- packages/firestore/src/local/local_store.ts | 149 +++++++++++------- .../src/local/memory_bundle_cache.ts | 10 +- .../test/unit/specs/bundle_spec.test.ts | 23 +++ .../test/unit/specs/spec_test_runner.ts | 6 +- yarn.lock | 143 +++++++++++++++-- 11 files changed, 264 insertions(+), 146 deletions(-) diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index 59584a6d82b..8e22d5de432 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -512,7 +512,7 @@ export class Firestore implements firestore.FirebaseFirestore, FirebaseService { return null; } - return new Query(namedQuery.query, this, null, namedQuery.readTime); + return new Query(namedQuery.query, this, null); } ensureClientConfigured(): FirestoreClient { @@ -1936,8 +1936,7 @@ export class Query implements firestore.Query { constructor( public _query: InternalQuery, readonly firestore: Firestore, - protected readonly _converter: firestore.FirestoreDataConverter | null, - private _readTime?: SnapshotVersion + protected readonly _converter: firestore.FirestoreDataConverter | null ) {} where( @@ -2236,7 +2235,6 @@ export class Query implements firestore.Query { firestoreClient, this._query, { - readFrom: this._readTime, includeMetadataChanges: options.includeMetadataChanges }, observer diff --git a/packages/firestore/src/core/event_manager.ts b/packages/firestore/src/core/event_manager.ts index f6265255b45..863cc317422 100644 --- a/packages/firestore/src/core/event_manager.ts +++ b/packages/firestore/src/core/event_manager.ts @@ -61,10 +61,7 @@ export class EventManager implements SyncEngineListener { this.syncEngine.subscribe(this); } - async listen( - listener: QueryListener, - readFrom?: SnapshotVersion - ): Promise { + async listen(listener: QueryListener): Promise { const query = listener.query; let firstListen = false; @@ -76,7 +73,7 @@ export class EventManager implements SyncEngineListener { if (firstListen) { try { - queryInfo.viewSnap = await this.syncEngine.listen(query, readFrom); + queryInfo.viewSnap = await this.syncEngine.listen(query); } catch (e) { const firestoreError = wrapInUserErrorIfRecoverable( e, @@ -200,16 +197,6 @@ export interface ListenOptions { * offline. */ readonly waitForSyncWhenOnline?: boolean; - - /** - * Tells the backend whether the client already has the query results up to - * a point in time, and the backend will only send deltas from that point - * on if applicable (for example, backend might already lose track of this - * particular query, and has to restart and send everything anyways). - * - * When not set, backend simply assumes the client does not have anything. - */ - readFrom?: SnapshotVersion | undefined; } /** diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index 16e51cae992..6b143118693 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -385,9 +385,7 @@ export class FirestoreClient { ): QueryListener { this.verifyNotTerminated(); const listener = new QueryListener(query, observer, options); - this.asyncQueue.enqueueAndForget(() => - this.eventMgr.listen(listener, options.readFrom) - ); + this.asyncQueue.enqueueAndForget(() => this.eventMgr.listen(listener)); return listener; } diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 5656edcc81f..5c69a1425fa 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -174,11 +174,8 @@ export interface SyncEngine extends RemoteSyncer { * Initiates the new listen, resolves promise when listen enqueued to the * server. All the subsequent view snapshots or errors are sent to the * subscribed handlers. Returns the initial snapshot. - * - * @param {SnapshotVersion} readFrom If provided, it tells the backend to - * only return deltas from this snapshot version on. */ - listen(query: Query, readFrom?: SnapshotVersion): Promise; + listen(query: Query): Promise; /** Stops listening to the query. */ unlisten(query: Query): Promise; @@ -320,10 +317,7 @@ class SyncEngineImpl implements SyncEngine { this.syncEngineListener = syncEngineListener; } - async listen( - query: Query, - readFrom?: SnapshotVersion - ): Promise { + async listen(query: Query): Promise { this.assertSubscribed('listen()'); let targetId; @@ -342,8 +336,7 @@ class SyncEngineImpl implements SyncEngine { viewSnapshot = queryView.view.computeInitialSnapshot(); } else { const targetData = await this.localStore.allocateTarget( - queryToTarget(query), - readFrom + queryToTarget(query) ); const status = this.sharedClientState.addLocalQueryTarget( diff --git a/packages/firestore/src/local/indexeddb_bundle_cache.ts b/packages/firestore/src/local/indexeddb_bundle_cache.ts index c8c107a5336..d9384b160d9 100644 --- a/packages/firestore/src/local/indexeddb_bundle_cache.ts +++ b/packages/firestore/src/local/indexeddb_bundle_cache.ts @@ -47,7 +47,7 @@ export class IndexedDbBundleCache implements BundleCache { .get(bundleId) .next(bundle => { if (bundle) { - return fromDbBundle(this.serializer, bundle); + return fromDbBundle(bundle); } return undefined; }); @@ -57,9 +57,7 @@ export class IndexedDbBundleCache implements BundleCache { transaction: PersistenceTransaction, bundleMetadata: bundleProto.BundleMetadata ): PersistencePromise { - return bundlesStore(transaction).put( - toDbBundle(this.serializer, bundleMetadata) - ); + return bundlesStore(transaction).put(toDbBundle(bundleMetadata)); } getNamedQuery( @@ -70,7 +68,7 @@ export class IndexedDbBundleCache implements BundleCache { .get(queryName) .next(query => { if (query) { - return fromDbNamedQuery(this.serializer, query); + return fromDbNamedQuery(query); } return undefined; }); @@ -80,9 +78,7 @@ export class IndexedDbBundleCache implements BundleCache { transaction: PersistenceTransaction, query: bundleProto.NamedQuery ): PersistencePromise { - return namedQueriesStore(transaction).put( - toDbNamedQuery(this.serializer, query) - ); + return namedQueriesStore(transaction).put(toDbNamedQuery(query)); } } diff --git a/packages/firestore/src/local/local_serializer.ts b/packages/firestore/src/local/local_serializer.ts index 7ea5bae157a..736a30bcf04 100644 --- a/packages/firestore/src/local/local_serializer.ts +++ b/packages/firestore/src/local/local_serializer.ts @@ -280,10 +280,7 @@ function isDocumentQuery(dbQuery: DbQuery): dbQuery is api.DocumentsTarget { } /** Encodes a DbBundle to a Bundle. */ -export function fromDbBundle( - serializer: LocalSerializer, - dbBundle: DbBundle -): Bundle { +export function fromDbBundle(dbBundle: DbBundle): Bundle { return { id: dbBundle.bundleId, createTime: fromDbTimestamp(dbBundle.createTime), @@ -292,10 +289,7 @@ export function fromDbBundle( } /** Encodes a BundleMetadata to a DbBundle. */ -export function toDbBundle( - serializer: LocalSerializer, - metadata: bundleProto.BundleMetadata -): DbBundle { +export function toDbBundle(metadata: bundleProto.BundleMetadata): DbBundle { return { bundleId: metadata.id!, createTime: toDbTimestamp(fromVersion(metadata.createTime!)), @@ -304,22 +298,16 @@ export function toDbBundle( } /** Encodes a DbNamedQuery to a NamedQuery. */ -export function fromDbNamedQuery( - serializer: LocalSerializer, - dbNamedQuery: DbNamedQuery -): NamedQuery { +export function fromDbNamedQuery(dbNamedQuery: DbNamedQuery): NamedQuery { return { name: dbNamedQuery.name, - query: fromBundledQuery(serializer, dbNamedQuery.bundledQuery), + query: fromBundledQuery(dbNamedQuery.bundledQuery), readTime: fromDbTimestamp(dbNamedQuery.readTime) }; } /** Encodes a NamedQuery from a bundle proto to a DbNamedQuery. */ -export function toDbNamedQuery( - serializer: LocalSerializer, - query: bundleProto.NamedQuery -): DbNamedQuery { +export function toDbNamedQuery(query: bundleProto.NamedQuery): DbNamedQuery { return { name: query.name!, readTime: toDbTimestamp(fromVersion(query.readTime!)), @@ -334,7 +322,6 @@ export function toDbNamedQuery( * including features exists only in SDKs (for example: limit-to-last). */ export function fromBundledQuery( - serializer: LocalSerializer, bundledQuery: bundleProto.BundledQuery ): Query { const query = convertQueryTargetToQuery({ @@ -353,19 +340,17 @@ export function fromBundledQuery( /** Encodes a NamedQuery proto object to a NamedQuery model object. */ export function fromProtoNamedQuery( - serializer: LocalSerializer, namedQuery: bundleProto.NamedQuery ): NamedQuery { return { name: namedQuery.name!, - query: fromBundledQuery(serializer, namedQuery.bundledQuery!), + query: fromBundledQuery(namedQuery.bundledQuery!), readTime: fromVersion(namedQuery.readTime!) }; } /** Encodes a BundleMetadata proto object to a Bundle model object. */ export function fromBundleMetadata( - serializer: LocalSerializer, metadata: bundleProto.BundleMetadata ): Bundle { return { diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 89feefd5fed..4827c0b5489 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -76,7 +76,8 @@ import { isIndexedDbTransactionError } from './simple_db'; import * as bundleProto from '../protos/firestore_bundle_proto'; import { BundleConverter, BundledDocuments, NamedQuery } from '../core/bundle'; import { BundleCache } from './bundle_cache'; -import { JsonProtoSerializer } from '../remote/serializer'; +import { fromVersion, JsonProtoSerializer } from '../remote/serializer'; +import { fromBundledQuery } from './local_serializer'; const LOG_TAG = 'LocalStore'; @@ -236,16 +237,20 @@ export interface LocalStore { * they don't get GC'd. A target must be allocated in the local store before * the store can be used to manage its view. * - * @param {SnapshotVersion} readFrom The snapshot version the allocated target + * @param {SnapshotVersion} readTime The snapshot version the allocated target * should resume from, unless the existing `TargetData` has an even newer * version. + * @param {PersistenceTransaction} txn The transaction to use if allocation is + * part of an existing transaction. A new transaction is created if this is + * absent. * * Allocating an already allocated `Target` will return the existing `TargetData` * for that `Target`. */ allocateTarget( target: Target, - readFrom?: SnapshotVersion + readTime?: SnapshotVersion, + txn?: PersistenceTransaction ): Promise; /** @@ -897,68 +902,84 @@ class LocalStoreImpl implements LocalStore { }); } - allocateTarget( + _allocateOperation( + txn: PersistenceTransaction, target: Target, - readFrom?: SnapshotVersion - ): Promise { - return this.persistence - .runTransaction('Allocate target', 'readwrite', txn => { - let targetData: TargetData; - return this.targetCache - .getTargetData(txn, target) - .next((cached: TargetData | null) => { - if (cached) { - // This target has been listened to previously, so reuse the - // previous targetID if no `readFrom` is provided. - // If `readFrom` is provided, the cached target data needs to be - // compared and updated if `readFrom` is newer. - // TODO(mcg): freshen last accessed date? - if (readFrom && cached.snapshotVersion.compareTo(readFrom) < 0) { - targetData = cached.withSnapshotVersion(readFrom); - return this.targetCache - .updateTargetData(txn, targetData) - .next(() => targetData); - } - targetData = cached; - return PersistencePromise.resolve(targetData); - } else { - return this.targetCache.allocateTargetId(txn).next(targetId => { - targetData = new TargetData( - target, - targetId, - TargetPurpose.Listen, - txn.currentSequenceNumber, - readFrom - ); - return this.targetCache - .addTargetData(txn, targetData) - .next(() => targetData); - }); - } + readTime?: SnapshotVersion + ): PersistencePromise { + let targetData: TargetData; + return this.targetCache + .getTargetData(txn, target) + .next((cached: TargetData | null) => { + if (cached) { + // This target has been listened to previously, so reuse the + // previous targetID if no `readTime` is provided. + // If `readTime` is provided, the cached target data needs to be + // compared and updated if `readTime` is newer. + // TODO(mcg): freshen last accessed date? + if (readTime && cached.snapshotVersion.compareTo(readTime) < 0) { + targetData = cached.withSnapshotVersion(readTime); + return this.targetCache + .updateTargetData(txn, targetData) + .next(() => targetData); + } + targetData = cached; + return PersistencePromise.resolve(targetData); + } else { + return this.targetCache.allocateTargetId(txn).next(targetId => { + targetData = new TargetData( + target, + targetId, + TargetPurpose.Listen, + txn.currentSequenceNumber, + readTime + ); + return this.targetCache + .addTargetData(txn, targetData) + .next(() => targetData); }); - }) - .then(targetData => { - // If Multi-Tab is enabled, the existing target data may be newer than - // the in-memory data - const cachedTargetData = this.targetDataByTarget.get( - targetData.targetId - ); - if ( - cachedTargetData === null || - targetData.snapshotVersion.compareTo( - cachedTargetData.snapshotVersion - ) > 0 - ) { - this.targetDataByTarget = this.targetDataByTarget.insert( - targetData.targetId, - targetData - ); - this.targetIdByTarget.set(target, targetData.targetId); } - return targetData; }); } + allocateTarget( + target: Target, + readTime?: SnapshotVersion, + txn?: PersistenceTransaction + ): Promise { + let allocateOperation: Promise; + if (txn) { + allocateOperation = this._allocateOperation( + txn, + target, + readTime + ).toPromise(); + } else { + allocateOperation = this.persistence.runTransaction( + 'Allocate target', + 'readwrite', + txn => this._allocateOperation(txn, target, readTime) + ); + } + return allocateOperation.then(targetData => { + // If Multi-Tab is enabled, the existing target data may be newer than + // the in-memory data + const cachedTargetData = this.targetDataByTarget.get(targetData.targetId); + if ( + cachedTargetData === null || + targetData.snapshotVersion.compareTo(cachedTargetData.snapshotVersion) > + 0 + ) { + this.targetDataByTarget = this.targetDataByTarget.insert( + targetData.targetId, + targetData + ); + this.targetIdByTarget.set(target, targetData.targetId); + } + return targetData; + }); + } + getTargetData( transaction: PersistenceTransaction, target: Target @@ -1398,6 +1419,14 @@ export function saveNamedQuery( return localStoreImpl.persistence.runTransaction( 'Save named query', 'readwrite', - transaction => localStoreImpl.bundleCache.saveNamedQuery(transaction, query) + transaction => { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + localStore.allocateTarget( + queryToTarget(fromBundledQuery(query.bundledQuery!)), + fromVersion(query.readTime!), + transaction + ); + return localStoreImpl.bundleCache.saveNamedQuery(transaction, query); + } ); } diff --git a/packages/firestore/src/local/memory_bundle_cache.ts b/packages/firestore/src/local/memory_bundle_cache.ts index b4ced5a38b2..788382ca387 100644 --- a/packages/firestore/src/local/memory_bundle_cache.ts +++ b/packages/firestore/src/local/memory_bundle_cache.ts @@ -43,10 +43,7 @@ export class MemoryBundleCache implements BundleCache { transaction: PersistenceTransaction, bundleMetadata: bundleProto.BundleMetadata ): PersistencePromise { - this.bundles.set( - bundleMetadata.id!, - fromBundleMetadata(this.serializer, bundleMetadata) - ); + this.bundles.set(bundleMetadata.id!, fromBundleMetadata(bundleMetadata)); return PersistencePromise.resolve(); } @@ -61,10 +58,7 @@ export class MemoryBundleCache implements BundleCache { transaction: PersistenceTransaction, query: bundleProto.NamedQuery ): PersistencePromise { - this.namedQueries.set( - query.name!, - fromProtoNamedQuery(this.serializer, query) - ); + this.namedQueries.set(query.name!, fromProtoNamedQuery(query)); return PersistencePromise.resolve(); } } diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts index 57a26edc55c..e3211c7e230 100644 --- a/packages/firestore/test/unit/specs/bundle_spec.test.ts +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -315,6 +315,29 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { }); }); + specTest('Bundles query can be resumed from same query.', [], () => { + const query1 = query('collection'); + const docA = doc('collection/a', 100, { key: 'a' }); + const bundleString1 = bundleWithDocumentAndQuery( + { + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { value: 'b' } + }, + { name: 'bundled-query', readTime: 400, query: query1 } + ); + + return spec() + .loadBundle(bundleString1) + .userListens(query1, 400) + .expectEvents(query1, { + added: [doc('collection/a', 500, { value: 'b' })], + fromCache: true + }); + }); + specTest( 'Bundles query can be loaded and resumed from different tabs', ['multi-client'], diff --git a/packages/firestore/test/unit/specs/spec_test_runner.ts b/packages/firestore/test/unit/specs/spec_test_runner.ts index 891145b939f..541377a651c 100644 --- a/packages/firestore/test/unit/specs/spec_test_runner.ts +++ b/packages/firestore/test/unit/specs/spec_test_runner.ts @@ -378,14 +378,12 @@ abstract class TestRunner { const querySpec = listenSpec.query; const query = parseQuery(querySpec); - let readFrom: SnapshotVersion; if (listenSpec.fromName) { const savedQuery = await getNamedQuery( this.localStore, listenSpec.fromName ); expect(queryEquals(query, savedQuery!.query)).to.be.true; - readFrom = savedQuery!.readTime; } const aggregator = new EventAggregator(query, e => { if (e.error) { @@ -401,9 +399,7 @@ abstract class TestRunner { const queryListener = new QueryListener(query, aggregator, options); this.queryListeners.set(query, queryListener); - await this.queue.enqueue(() => - this.eventManager.listen(queryListener, readFrom) - ); + await this.queue.enqueue(() => this.eventManager.listen(queryListener)); if (targetFailed) { expect(this.persistence.injectFailures).contains('Allocate target'); diff --git a/yarn.lock b/yarn.lock index d76b22c3bd6..8f8748d40b3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -79,6 +79,15 @@ lodash "^4.17.13" source-map "^0.5.0" +"@babel/generator@^7.11.0", "@babel/generator@^7.4.0": + version "7.11.0" + resolved "https://registry.npmjs.org/@babel/generator/-/generator-7.11.0.tgz#4b90c78d8c12825024568cbe83ee6c9af193585c" + integrity sha512-fEm3Uzw7Mc9Xi//qU20cBKatTfs2aOtKqmvy/Vm7RkJEGFQ4xc9myCfbXxqK//ZS8MR/ciOHw6meGASJuKmDfQ== + dependencies: + "@babel/types" "^7.11.0" + jsesc "^2.5.1" + source-map "^0.5.0" + "@babel/generator@^7.9.0": version "7.9.4" resolved "https://registry.npmjs.org/@babel/generator/-/generator-7.9.4.tgz#12441e90c3b3c4159cdecf312075bf1a8ce2dbce" @@ -354,6 +363,13 @@ dependencies: "@babel/types" "^7.10.4" +"@babel/helper-split-export-declaration@^7.11.0": + version "7.11.0" + resolved "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.11.0.tgz#f8a491244acf6a676158ac42072911ba83ad099f" + integrity sha512-74Vejvp6mHkGE+m+k5vHY93FX2cAtrw1zXrZXRlG4l410Nm9PxfEiVTn1PjDPV5SnmieiueY4AFg2xqhNFuuZg== + dependencies: + "@babel/types" "^7.11.0" + "@babel/helper-split-export-declaration@^7.8.3": version "7.8.3" resolved "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.8.3.tgz#31a9f30070f91368a7182cf05f831781065fc7a9" @@ -427,6 +443,11 @@ resolved "https://registry.npmjs.org/@babel/parser/-/parser-7.10.4.tgz#9eedf27e1998d87739fb5028a5120557c06a1a64" integrity sha512-8jHII4hf+YVDsskTF6WuMB3X4Eh+PsUkC2ljq22so5rHvH+T8BzyL94VOdyFLNR8tBSVXOTbNHOKpR4TfRxVtA== +"@babel/parser@^7.11.0", "@babel/parser@^7.4.3": + version "7.11.0" + resolved "https://registry.npmjs.org/@babel/parser/-/parser-7.11.0.tgz#a9d7e11aead25d3b422d17b2c6502c8dddef6a5d" + integrity sha512-qvRvi4oI8xii8NllyEc4MDJjuZiNaRzyb7Y7lup1NqJV8TZHF4O27CcP+72WPn/k1zkgJ6WJfnIbk4jTsVAZHw== + "@babel/parser@^7.7.5", "@babel/parser@^7.8.6", "@babel/parser@^7.9.0": version "7.9.4" resolved "https://registry.npmjs.org/@babel/parser/-/parser-7.9.4.tgz#68a35e6b0319bbc014465be43828300113f2f2e8" @@ -962,7 +983,7 @@ dependencies: regenerator-runtime "^0.13.4" -"@babel/template@^7.10.4": +"@babel/template@^7.10.4", "@babel/template@^7.4.0": version "7.10.4" resolved "https://registry.npmjs.org/@babel/template/-/template-7.10.4.tgz#3251996c4200ebc71d1a8fc405fba940f36ba278" integrity sha512-ZCjD27cGJFUB6nmCB1Enki3r+L5kJveX9pq1SvAUKoICy6CZ9yD8xO086YXdYhvNjBdnekm4ZnaP5yC8Cs/1tA== @@ -995,6 +1016,21 @@ globals "^11.1.0" lodash "^4.17.13" +"@babel/traverse@^7.4.3": + version "7.11.0" + resolved "https://registry.npmjs.org/@babel/traverse/-/traverse-7.11.0.tgz#9b996ce1b98f53f7c3e4175115605d56ed07dd24" + integrity sha512-ZB2V+LskoWKNpMq6E5UUCrjtDUh5IOTAyIl0dTjIEoXum/iKWkoIEKIRDnUucO6f+2FzNkE0oD4RLKoPIufDtg== + dependencies: + "@babel/code-frame" "^7.10.4" + "@babel/generator" "^7.11.0" + "@babel/helper-function-name" "^7.10.4" + "@babel/helper-split-export-declaration" "^7.11.0" + "@babel/parser" "^7.11.0" + "@babel/types" "^7.11.0" + debug "^4.1.0" + globals "^11.1.0" + lodash "^4.17.19" + "@babel/traverse@^7.7.4", "@babel/traverse@^7.8.6", "@babel/traverse@^7.9.0": version "7.9.0" resolved "https://registry.npmjs.org/@babel/traverse/-/traverse-7.9.0.tgz#d3882c2830e513f4fe4cec9fe76ea1cc78747892" @@ -1019,6 +1055,15 @@ lodash "^4.17.13" to-fast-properties "^2.0.0" +"@babel/types@^7.11.0", "@babel/types@^7.4.0": + version "7.11.0" + resolved "https://registry.npmjs.org/@babel/types/-/types-7.11.0.tgz#2ae6bf1ba9ae8c3c43824e5861269871b206e90d" + integrity sha512-O53yME4ZZI0jO1EVGtF1ePGl0LHirG4P1ibcD80XyzZcKhcMFeCXmh4Xb1ifGBIV233Qg12x4rBfQgA+tmOukA== + dependencies: + "@babel/helper-validator-identifier" "^7.10.4" + lodash "^4.17.19" + to-fast-properties "^2.0.0" + "@babel/types@^7.4.4": version "7.9.5" resolved "https://registry.npmjs.org/@babel/types/-/types-7.9.5.tgz#89231f82915a8a566a703b3b20133f73da6b9444" @@ -3260,6 +3305,13 @@ append-buffer@^1.0.2: dependencies: buffer-equal "^1.0.0" +append-transform@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/append-transform/-/append-transform-1.0.0.tgz#046a52ae582a228bd72f58acfbe2967c678759ab" + integrity sha512-P009oYkeHyU742iSZJzZZywj4QRJdnTWffaKuJQLablCZ1uz6/cW4yaRgcDaoQ+uwOxxnt0gRUcwfsNP2ri0gw== + dependencies: + default-require-extensions "^2.0.0" + append-transform@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/append-transform/-/append-transform-2.0.0.tgz#99d9d29c7b38391e6f428d28ce136551f0b77e12" @@ -4908,7 +4960,7 @@ compare-semver@^1.0.0: dependencies: semver "^5.0.1" -compare-versions@^3.6.0: +compare-versions@^3.4.0, compare-versions@^3.6.0: version "3.6.0" resolved "https://registry.npmjs.org/compare-versions/-/compare-versions-3.6.0.tgz#1a5689913685e5a87637b8d3ffca75514ec41d62" integrity sha512-W6Af2Iw1z4CB7q4uU4hv646dW9GQuBM+YpC0UvUCWSD8w90SJjp+ujJuXaEMtAXBtSqGfMPuFOVn4/+FlaqfBA== @@ -5631,6 +5683,13 @@ default-compare@^1.0.0: dependencies: kind-of "^5.0.2" +default-require-extensions@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-2.0.0.tgz#f5f8fbb18a7d6d50b21f641f649ebb522cfe24f7" + integrity sha1-9fj7sYp9bVCyH2QfZJ67Uiz+JPc= + dependencies: + strip-bom "^3.0.0" + default-require-extensions@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-3.0.0.tgz#e03f93aac9b2b6443fc52e5e4a37b3ad9ad8df96" @@ -6806,6 +6865,14 @@ file-uri-to-path@1.0.0: resolved "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== +fileset@^2.0.3: + version "2.0.3" + resolved "https://registry.npmjs.org/fileset/-/fileset-2.0.3.tgz#8e7548a96d3cc2327ee5e674168723a333bba2a0" + integrity sha1-jnVIqW08wjJ+5eZ0FocjozO7oqA= + dependencies: + glob "^7.0.3" + minimatch "^3.0.3" + filesize@^3.1.3: version "3.6.1" resolved "https://registry.npmjs.org/filesize/-/filesize-3.6.1.tgz#090bb3ee01b6f801a8a8be99d31710b3422bb317" @@ -9063,6 +9130,25 @@ isstream@~0.1.2: resolved "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo= +istanbul-api@^2.1.6: + version "2.1.7" + resolved "https://registry.npmjs.org/istanbul-api/-/istanbul-api-2.1.7.tgz#82786b79f3b93d481349c7aa1e2c2b4eeb48c8a8" + integrity sha512-LYTOa2UrYFyJ/aSczZi/6lBykVMjCCvUmT64gOe+jPZFy4w6FYfPGqFT2IiQ2BxVHHDOvCD7qrIXb0EOh4uGWw== + dependencies: + async "^2.6.2" + compare-versions "^3.4.0" + fileset "^2.0.3" + istanbul-lib-coverage "^2.0.5" + istanbul-lib-hook "^2.0.7" + istanbul-lib-instrument "^3.3.0" + istanbul-lib-report "^2.0.8" + istanbul-lib-source-maps "^3.0.6" + istanbul-reports "^2.2.5" + js-yaml "^3.13.1" + make-dir "^2.1.0" + minimatch "^3.0.4" + once "^1.4.0" + istanbul-instrumenter-loader@3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/istanbul-instrumenter-loader/-/istanbul-instrumenter-loader-3.0.1.tgz#9957bd59252b373fae5c52b7b5188e6fde2a0949" @@ -9088,6 +9174,13 @@ istanbul-lib-coverage@^3.0.0, istanbul-lib-coverage@^3.0.0-alpha.1: resolved "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.0.0.tgz#f5944a37c70b550b02a78a5c3b2055b280cec8ec" integrity sha512-UiUIqxMgRDET6eR+o5HbfRYP1l0hqkWOs7vNxC/mggutCMUIhWMm8gAHb8tHlyfD3/l6rlgNA5cKdDzEAf6hEg== +istanbul-lib-hook@^2.0.7: + version "2.0.7" + resolved "https://registry.npmjs.org/istanbul-lib-hook/-/istanbul-lib-hook-2.0.7.tgz#c95695f383d4f8f60df1f04252a9550e15b5b133" + integrity sha512-vrRztU9VRRFDyC+aklfLoeXyNdTfga2EI3udDGn4cZ6fpSXpHLV9X6CHvfoMCPtggg8zvDDmC4b9xfu0z6/llA== + dependencies: + append-transform "^1.0.0" + istanbul-lib-hook@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/istanbul-lib-hook/-/istanbul-lib-hook-3.0.0.tgz#8f84c9434888cc6b1d0a9d7092a76d239ebf0cc6" @@ -9108,6 +9201,19 @@ istanbul-lib-instrument@^1.7.3: istanbul-lib-coverage "^1.2.1" semver "^5.3.0" +istanbul-lib-instrument@^3.3.0: + version "3.3.0" + resolved "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-3.3.0.tgz#a5f63d91f0bbc0c3e479ef4c5de027335ec6d630" + integrity sha512-5nnIN4vo5xQZHdXno/YDXJ0G+I3dAm4XgzfSVTPLQpj/zAV2dV6Juy0yaf10/zrJOJeHoN3fraFe+XRq2bFVZA== + dependencies: + "@babel/generator" "^7.4.0" + "@babel/parser" "^7.4.3" + "@babel/template" "^7.4.0" + "@babel/traverse" "^7.4.3" + "@babel/types" "^7.4.0" + istanbul-lib-coverage "^2.0.5" + semver "^6.0.0" + istanbul-lib-instrument@^4.0.0: version "4.0.1" resolved "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-4.0.1.tgz#61f13ac2c96cfefb076fe7131156cc05907874e6" @@ -9134,6 +9240,15 @@ istanbul-lib-processinfo@^2.0.2: rimraf "^3.0.0" uuid "^3.3.3" +istanbul-lib-report@^2.0.8: + version "2.0.8" + resolved "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-2.0.8.tgz#5a8113cd746d43c4889eba36ab10e7d50c9b4f33" + integrity sha512-fHBeG573EIihhAblwgxrSenp0Dby6tJMFR/HvlerBsrCTD5bkUuoNtn3gVh29ZCS824cGGBPn7Sg7cNk+2xUsQ== + dependencies: + istanbul-lib-coverage "^2.0.5" + make-dir "^2.1.0" + supports-color "^6.1.0" + istanbul-lib-report@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz#7518fe52ea44de372f460a76b5ecda9ffb73d8a6" @@ -9163,6 +9278,13 @@ istanbul-lib-source-maps@^4.0.0: istanbul-lib-coverage "^3.0.0" source-map "^0.6.1" +istanbul-reports@^2.2.5: + version "2.2.7" + resolved "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-2.2.7.tgz#5d939f6237d7b48393cc0959eab40cd4fd056931" + integrity sha512-uu1F/L1o5Y6LzPVSVZXNOoD/KXpJue9aeLRd0sM9uMXfZvzomB0WxVamWb5ue8kA2vVWEmW7EG+A5n3f1kqHKg== + dependencies: + html-escaper "^2.0.0" + istanbul-reports@^3.0.0, istanbul-reports@^3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.0.2.tgz#d593210e5000683750cb09fc0644e4b6e27fd53b" @@ -9472,15 +9594,12 @@ karma-cli@2.0.0: dependencies: resolve "^1.3.3" -karma-coverage-istanbul-reporter@3.0.3: - version "3.0.3" - resolved "https://registry.npmjs.org/karma-coverage-istanbul-reporter/-/karma-coverage-istanbul-reporter-3.0.3.tgz#f3b5303553aadc8e681d40d360dfdc19bc7e9fe9" - integrity sha512-wE4VFhG/QZv2Y4CdAYWDbMmcAHeS926ZIji4z+FkB2aF/EposRb6DP6G5ncT/wXhqUfAb/d7kZrNKPonbvsATw== +karma-coverage-istanbul-reporter@2.1.1: + version "2.1.1" + resolved "https://registry.npmjs.org/karma-coverage-istanbul-reporter/-/karma-coverage-istanbul-reporter-2.1.1.tgz#37a775fbfbb3cbe98cebf19605c94c6277c3b88a" + integrity sha512-CH8lTi8+kKXGvrhy94+EkEMldLCiUA0xMOiL31vvli9qK0T+qcXJAwWBRVJWnVWxYkTmyWar8lPz63dxX6/z1A== dependencies: - istanbul-lib-coverage "^3.0.0" - istanbul-lib-report "^3.0.0" - istanbul-lib-source-maps "^3.0.6" - istanbul-reports "^3.0.2" + istanbul-api "^2.1.6" minimatch "^3.0.4" karma-firefox-launcher@1.3.0: @@ -10157,7 +10276,7 @@ lodash@4.17.15, lodash@^4.0.0, lodash@^4.16.6, lodash@^4.17.10, lodash@^4.17.11, resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A== -lodash@4.17.19: +lodash@4.17.19, lodash@^4.17.19: version "4.17.19" resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz#e48ddedbe30b3321783c5b4301fbd353bc1e4a4b" integrity sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ== @@ -10611,7 +10730,7 @@ minimalistic-crypto-utils@^1.0.0, minimalistic-crypto-utils@^1.0.1: resolved "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz#f6c00c1c0b082246e5c4d99dfb8c7c083b2b582a" integrity sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo= -minimatch@3.0.4, minimatch@^3.0.0, minimatch@^3.0.4: +minimatch@3.0.4, minimatch@^3.0.0, minimatch@^3.0.3, minimatch@^3.0.4: version "3.0.4" resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== From 8ea9e7b6e2a8f346fd6488fb0d10be4a941ba8b9 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 31 Jul 2020 14:43:29 -0400 Subject: [PATCH 39/43] Save named queries to target cache as well. --- packages/firestore/exp/src/api/database.ts | 7 +- packages/firestore/src/api/database.ts | 1 - packages/firestore/src/core/bundle.ts | 4 +- packages/firestore/src/core/event_manager.ts | 1 - packages/firestore/src/local/local_store.ts | 2 +- packages/firestore/src/local/target_data.ts | 4 +- .../integration/api_internal/bundle.test.ts | 14 +-- .../test/unit/local/local_store.test.ts | 90 ++++++++++++++++--- .../test/unit/specs/bundle_spec.test.ts | 27 +----- .../firestore/test/unit/specs/spec_builder.ts | 11 --- .../test/unit/specs/spec_test_runner.ts | 11 +-- packages/firestore/test/util/helpers.ts | 24 ++++- 12 files changed, 119 insertions(+), 77 deletions(-) diff --git a/packages/firestore/exp/src/api/database.ts b/packages/firestore/exp/src/api/database.ts index 52f80f80a7f..20e9d489a58 100644 --- a/packages/firestore/exp/src/api/database.ts +++ b/packages/firestore/exp/src/api/database.ts @@ -44,13 +44,14 @@ import { cast } from '../../../lite/src/api/util'; import { Code, FirestoreError } from '../../../src/util/error'; import { Deferred } from '../../../src/util/promise'; import { LruParams } from '../../../src/local/lru_garbage_collector'; -import { CACHE_SIZE_UNLIMITED, Query } from '../../../src/api/database'; +import { CACHE_SIZE_UNLIMITED } from '../../../src/api/database'; import { DatabaseId, DatabaseInfo } from '../../../src/core/database_info'; import { indexedDbStoragePrefix, indexedDbClearPersistence } from '../../../src/local/indexeddb_persistence'; import { LoadBundleTask } from '../../../src/api/bundle'; +import { Query } from '../../../lite'; /** * The root reference to the Firestore database and the entry point for the @@ -322,7 +323,5 @@ export async function namedQuery( return null; } - // TODO(wuandy): make this work with exp build. - return null; - // return new Query(namedQuery.query, firestoreImpl, null, namedQuery.readTime); + return new Query(firestoreImpl, null, namedQuery.query, namedQuery.readTime); } diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index 8e22d5de432..ebac7a323cd 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -112,7 +112,6 @@ import { import { UserDataWriter } from './user_data_writer'; import { FirebaseAuthInternalName } from '@firebase/auth-interop-types'; import { Provider } from '@firebase/component'; -import { SnapshotVersion } from '../core/snapshot_version'; import { LoadBundleTask } from './bundle'; // settings() defaults: diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index d1a6ca23d0c..55f00ffb39b 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -180,11 +180,9 @@ export class BundleLoader { this.progress.bytesLoaded += element.byteLength; let documentsLoaded = this.progress.documentsLoaded; - let loadedNamedQuery = false; if (element.payload.namedQuery) { this.queries.push(element.payload.namedQuery); - loadedNamedQuery = true; } else if (element.payload.documentMetadata) { this.documents.push({ metadata: element.payload.documentMetadata }); if (!element.payload.documentMetadata.exists) { @@ -202,7 +200,7 @@ export class BundleLoader { ++documentsLoaded; } - if (loadedNamedQuery || documentsLoaded !== this.progress.documentsLoaded) { + if (documentsLoaded !== this.progress.documentsLoaded) { this.progress.documentsLoaded = documentsLoaded; return { ...this.progress }; } diff --git a/packages/firestore/src/core/event_manager.ts b/packages/firestore/src/core/event_manager.ts index 863cc317422..c3823e2de98 100644 --- a/packages/firestore/src/core/event_manager.ts +++ b/packages/firestore/src/core/event_manager.ts @@ -23,7 +23,6 @@ import { SyncEngine, SyncEngineListener } from './sync_engine'; import { OnlineState } from './types'; import { ChangeType, DocumentViewChange, ViewSnapshot } from './view_snapshot'; import { wrapInUserErrorIfRecoverable } from '../util/async_queue'; -import { SnapshotVersion } from './snapshot_version'; /** * Holds the listeners and the last received ViewSnapshot for a query being diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 4827c0b5489..6cc75c1491e 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -918,7 +918,7 @@ class LocalStoreImpl implements LocalStore { // compared and updated if `readTime` is newer. // TODO(mcg): freshen last accessed date? if (readTime && cached.snapshotVersion.compareTo(readTime) < 0) { - targetData = cached.withSnapshotVersion(readTime); + targetData = cached.withReadTime(readTime); return this.targetCache .updateTargetData(txn, targetData) .next(() => targetData); diff --git a/packages/firestore/src/local/target_data.ts b/packages/firestore/src/local/target_data.ts index 19ffa22a654..fad5aeec763 100644 --- a/packages/firestore/src/local/target_data.ts +++ b/packages/firestore/src/local/target_data.ts @@ -69,8 +69,8 @@ export class TargetData { readonly resumeToken: ByteString = ByteString.EMPTY_BYTE_STRING ) {} - /** Creates a new target data instance with an updated sequence number. */ - withSnapshotVersion(version: SnapshotVersion): TargetData { + /** Creates a new target data instance with an updated read time. */ + withReadTime(version: SnapshotVersion): TargetData { return new TargetData( this.target, this.targetId, diff --git a/packages/firestore/test/integration/api_internal/bundle.test.ts b/packages/firestore/test/integration/api_internal/bundle.test.ts index 24823f12c40..74e27189e8c 100644 --- a/packages/firestore/test/integration/api_internal/bundle.test.ts +++ b/packages/firestore/test/integration/api_internal/bundle.test.ts @@ -30,6 +30,8 @@ import { TestBundleBuilder } from '../../unit/util/bundle_data'; import { newTextEncoder } from '../../../src/platform/serializer'; import { collectionReference } from '../../util/api_helpers'; +// TODO(b/162594908): Move this to api/ instead of api_internal. + export const encoder = newTextEncoder(); function verifySuccessProgress(p: firestore.LoadBundleTaskProgress): void { @@ -132,14 +134,12 @@ apiDescribe('Bundles', (persistence: boolean) => { }); expect(completeCalled).to.be.true; - expect(progressEvents.length).to.equal(6); + expect(progressEvents.length).to.equal(4); verifyInProgress(progressEvents[0], 0); - verifyInProgress(progressEvents[1], 0); - verifyInProgress(progressEvents[2], 0); - verifyInProgress(progressEvents[3], 1); - verifyInProgress(progressEvents[4], 2); - verifySuccessProgress(progressEvents[5]); - expect(fulfillProgress!).to.deep.equal(progressEvents[5]); + verifyInProgress(progressEvents[1], 1); + verifyInProgress(progressEvents[2], 2); + verifySuccessProgress(progressEvents[3]); + expect(fulfillProgress!).to.deep.equal(progressEvents[3]); // Read from cache. These documents do not exist in backend, so they can // only be read from cache. diff --git a/packages/firestore/test/unit/local/local_store.test.ts b/packages/firestore/test/unit/local/local_store.test.ts index 2ff32770de0..d594c4d3de6 100644 --- a/packages/firestore/test/unit/local/local_store.test.ts +++ b/packages/firestore/test/unit/local/local_store.test.ts @@ -21,7 +21,13 @@ import { expect } from 'chai'; import { FieldValue } from '../../../src/api/field_value'; import { Timestamp } from '../../../src/api/timestamp'; import { User } from '../../../src/auth/user'; -import { Query, queryToTarget } from '../../../src/core/query'; +import { + LimitType, + Query, + queryEquals, + queryToTarget, + queryWithLimit +} from '../../../src/core/query'; import { Target } from '../../../src/core/target'; import { BatchId, TargetId } from '../../../src/core/types'; import { SnapshotVersion } from '../../../src/core/snapshot_version'; @@ -29,11 +35,13 @@ import { IndexFreeQueryEngine } from '../../../src/local/index_free_query_engine import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; import { applyBundleDocuments, + getNamedQuery, hasNewerBundle, LocalStore, LocalWriteResult, - saveBundle, newLocalStore, + saveBundle, + saveNamedQuery, synchronizeLastDocumentChangeReadTime } from '../../../src/local/local_store'; import { LocalViewChanges } from '../../../src/local/local_view_changes'; @@ -63,6 +71,8 @@ import { import { debugAssert } from '../../../src/util/assert'; import { addEqualityMatcher } from '../../util/equality_matcher'; import { + bundledDocuments, + bundleMetadata, byteStringFromString, deletedDoc, deleteMutation, @@ -74,25 +84,25 @@ import { key, localViewChanges, mapAsArray, + namedQuery, noChangeEvent, + orderBy, patchMutation, query, setMutation, - bundledDocuments, TestBundledDocuments, TestSnapshotVersion, transformMutation, unknownDoc, - version, - bundleMetadata + version } from '../../util/helpers'; import { CountingQueryEngine, QueryEngineType } from './counting_query_engine'; import * as persistenceHelpers from './persistence_test_helpers'; -import { ByteString } from '../../../src/util/byte_string'; -import { BundledDocuments } from '../../../src/core/bundle'; import { JSON_SERIALIZER } from './persistence_test_helpers'; -import { BundleMetadata } from '../../../src/protos/firestore_bundle_proto'; +import { ByteString } from '../../../src/util/byte_string'; +import { BundledDocuments, NamedQuery } from '../../../src/core/bundle'; +import * as bundleProto from '../../../src/protos/firestore_bundle_proto'; export interface LocalStoreComponents { queryEngine: CountingQueryEngine; @@ -127,6 +137,7 @@ class LocalStoreTester { | RemoteEvent | LocalViewChanges | TestBundledDocuments + | bundleProto.NamedQuery ): LocalStoreTester { if (op instanceof Mutation) { return this.afterMutations([op]); @@ -136,8 +147,10 @@ class LocalStoreTester { return this.afterViewChanges(op); } else if (op instanceof RemoteEvent) { return this.afterRemoteEvent(op); - } else { + } else if (op instanceof TestBundledDocuments) { return this.afterBundleDocuments(op.documents); + } else { + return this.afterNamedQuery(op); } } @@ -181,6 +194,15 @@ class LocalStoreTester { return this; } + afterNamedQuery(namedQuery: bundleProto.NamedQuery): LocalStoreTester { + this.prepareNextStep(); + + this.promiseChain = this.promiseChain.then(() => + saveNamedQuery(this.localStore, namedQuery) + ); + return this; + } + afterViewChanges(viewChanges: LocalViewChanges): LocalStoreTester { this.prepareNextStep(); @@ -411,7 +433,7 @@ class LocalStoreTester { } toHaveNewerBundle( - metadata: BundleMetadata, + metadata: bundleProto.BundleMetadata, expected: boolean ): LocalStoreTester { this.promiseChain = this.promiseChain.then(() => { @@ -422,7 +444,19 @@ class LocalStoreTester { return this; } - afterSavingBundle(metadata: BundleMetadata): LocalStoreTester { + toHaveNamedQuery(namedQuery: NamedQuery): LocalStoreTester { + this.promiseChain = this.promiseChain.then(() => { + return getNamedQuery(this.localStore, namedQuery.name).then(actual => { + expect(!!actual).to.be.true; + expect(actual!.name).to.equal(namedQuery.name); + expect(namedQuery.readTime.isEqual(actual!.readTime)).to.be.true; + expect(queryEquals(actual!.query, namedQuery.query)).to.be.true; + }); + }); + return this; + } + + afterSavingBundle(metadata: bundleProto.BundleMetadata): LocalStoreTester { this.promiseChain = this.promiseChain.then(() => saveBundle(this.localStore, metadata) ); @@ -1654,6 +1688,40 @@ function genericLocalStoreTests( .finish(); }); + it('handles saving and loading named queries', async () => { + return expectLocalStore() + .after(namedQuery('test', query('coll'), 'FIRST', SnapshotVersion.min())) + .toHaveNamedQuery({ + name: 'test', + query: query('coll'), + readTime: SnapshotVersion.min() + }) + .finish(); + }); + + it('handles saving and loading limit to last queries', async () => { + const now = Timestamp.now(); + return expectLocalStore() + .after( + namedQuery( + 'test', + queryWithLimit(query('coll', orderBy('sort')), 5, LimitType.First), + 'LAST', + SnapshotVersion.fromTimestamp(now) + ) + ) + .toHaveNamedQuery({ + name: 'test', + query: queryWithLimit( + query('coll', orderBy('sort')), + 5, + LimitType.Last + ), + readTime: SnapshotVersion.fromTimestamp(now) + }) + .finish(); + }); + it('computes highest unacknowledged batch id correctly', () => { return expectLocalStore() .toReturnHighestUnacknowledgeBatchId(BATCHID_UNKNOWN) diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts index e3211c7e230..5b0ced3ae9b 100644 --- a/packages/firestore/test/unit/specs/bundle_spec.test.ts +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -292,29 +292,6 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { ); }); - specTest('Bundles query can be loaded and resumed.', [], () => { - const query1 = query('collection'); - const docA = doc('collection/a', 100, { key: 'a' }); - const bundleString1 = bundleWithDocumentAndQuery( - { - key: docA.key, - readTime: 500, - createTime: 250, - updateTime: 500, - content: { value: 'b' } - }, - { name: 'bundled-query', readTime: 400, query: query1 } - ); - - return spec() - .loadBundle(bundleString1) - .userListensToNamedQuery('bundled-query', query1, 400) - .expectEvents(query1, { - added: [doc('collection/a', 500, { value: 'b' })], - fromCache: true - }); - }); - specTest('Bundles query can be resumed from same query.', [], () => { const query1 = query('collection'); const docA = doc('collection/a', 100, { key: 'a' }); @@ -372,7 +349,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { .loadBundle(bundleString1) // Read named query from loaded bundle by primary. .client(1) - .userListensToNamedQuery('bundled-query', query1, 400) + .userListens(query1, 400) .expectEvents(query1, { added: [doc('collection/a', 500, { value: 'b' })], fromCache: true @@ -386,7 +363,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { .userUnlistens(query1) // Read named query from loaded bundle by secondary. .client(0) - .userListensToNamedQuery('bundled-query', query2, 560) + .userListens(query2, 560) .expectEvents(query2, { added: [doc('collection/a', 550, { value: 'c' })], fromCache: true diff --git a/packages/firestore/test/unit/specs/spec_builder.ts b/packages/firestore/test/unit/specs/spec_builder.ts index b4ca45fd1c0..79b69ec2802 100644 --- a/packages/firestore/test/unit/specs/spec_builder.ts +++ b/packages/firestore/test/unit/specs/spec_builder.ts @@ -369,17 +369,6 @@ export class SpecBuilder { return this; } - userListensToNamedQuery( - name: string, - query: Query, - readFrom: TestSnapshotVersion - ): this { - // Note we do not call this.nextStep() here because we reuse userListens(). - this.userListens(query, readFrom); - this.currentStep!.userListen!.fromName = name; - return this; - } - // PORTING NOTE: Only used by web multi-tab tests. becomeHidden(): this { this.nextStep(); diff --git a/packages/firestore/test/unit/specs/spec_test_runner.ts b/packages/firestore/test/unit/specs/spec_test_runner.ts index 541377a651c..e58683db68a 100644 --- a/packages/firestore/test/unit/specs/spec_test_runner.ts +++ b/packages/firestore/test/unit/specs/spec_test_runner.ts @@ -49,7 +49,7 @@ import { SCHEMA_VERSION, SchemaConverter } from '../../../src/local/indexeddb_schema'; -import { getNamedQuery, LocalStore } from '../../../src/local/local_store'; +import { LocalStore } from '../../../src/local/local_store'; import { ClientId, SharedClientState @@ -378,13 +378,7 @@ abstract class TestRunner { const querySpec = listenSpec.query; const query = parseQuery(querySpec); - if (listenSpec.fromName) { - const savedQuery = await getNamedQuery( - this.localStore, - listenSpec.fromName - ); - expect(queryEquals(query, savedQuery!.query)).to.be.true; - } + const aggregator = new EventAggregator(query, e => { if (e.error) { targetFailed = true; @@ -1376,7 +1370,6 @@ export interface SpecStep { export interface SpecUserListen { targetId: TargetId; query: string | SpecQuery; - fromName?: string; } /** [, ] */ diff --git a/packages/firestore/test/util/helpers.ts b/packages/firestore/test/util/helpers.ts index 2f9e50f801b..fa358ab76f7 100644 --- a/packages/firestore/test/util/helpers.ts +++ b/packages/firestore/test/util/helpers.ts @@ -101,6 +101,8 @@ import { JsonProtoSerializer, toDocument, toName, + toQueryTarget, + toTimestamp, toVersion } from '../../src/remote/serializer'; import { Timestamp } from '../../src/api/timestamp'; @@ -112,7 +114,7 @@ import { TEST_DATABASE_ID } from '../unit/local/persistence_test_helpers'; import { BundledDocuments } from '../../src/core/bundle'; -import { BundleMetadata } from '../../src/protos/firestore_bundle_proto'; +import * as bundleProto from '../../src/protos/firestore_bundle_proto'; /* eslint-disable no-restricted-globals */ @@ -460,13 +462,31 @@ export function bundledDocuments( return new TestBundledDocuments(result); } +export function namedQuery( + name: string, + query: Query, + limitType: bundleProto.LimitType, + readTime: SnapshotVersion +): bundleProto.NamedQuery { + return { + name, + readTime: toTimestamp(JSON_SERIALIZER, readTime.toTimestamp()), + bundledQuery: { + parent: toQueryTarget(JSON_SERIALIZER, queryToTarget(query)).parent, + limitType, + structuredQuery: toQueryTarget(JSON_SERIALIZER, queryToTarget(query)) + .structuredQuery + } + }; +} + export function bundleMetadata( id: string, createTime: TestSnapshotVersion, version = 1, totalDocuments = 1, totalBytes = 1000 -): BundleMetadata { +): bundleProto.BundleMetadata { return { id, createTime: { seconds: createTime, nanos: 0 }, From 487982ca887b04fc405203c9b58350c2d638964f Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 31 Jul 2020 14:49:56 -0400 Subject: [PATCH 40/43] Fix more errors. --- packages/firestore/lite/src/api/reference.ts | 6 +- packages/firestore/src/api/database.ts | 4 +- yarn.lock | 143 ++----------------- 3 files changed, 15 insertions(+), 138 deletions(-) diff --git a/packages/firestore/lite/src/api/reference.ts b/packages/firestore/lite/src/api/reference.ts index 42153108fb0..fa734b4b700 100644 --- a/packages/firestore/lite/src/api/reference.ts +++ b/packages/firestore/lite/src/api/reference.ts @@ -81,7 +81,6 @@ import { import { newSerializer } from '../../../src/platform/serializer'; import { FieldPath as ExternalFieldPath } from '../../../src/api/field_path'; import { Code, FirestoreError } from '../../../src/util/error'; -import { SnapshotVersion } from '../../../src/core/snapshot_version'; /** * A reference to a particular document in a collection in the database. @@ -121,14 +120,13 @@ export class Query implements firestore.Query { constructor( readonly firestore: Firestore, readonly converter: firestore.FirestoreDataConverter | null, - readonly _query: InternalQuery, - readonly _readFrom?: SnapshotVersion + readonly _query: InternalQuery ) {} withConverter( converter: firestore.FirestoreDataConverter ): firestore.Query { - return new Query(this.firestore, converter, this._query, this._readFrom); + return new Query(this.firestore, converter, this._query); } } diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index ebac7a323cd..7d48f4252ae 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -2233,9 +2233,7 @@ export class Query implements firestore.Query { return addQuerySnapshotListener( firestoreClient, this._query, - { - includeMetadataChanges: options.includeMetadataChanges - }, + options, observer ); } diff --git a/yarn.lock b/yarn.lock index 8f8748d40b3..d76b22c3bd6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -79,15 +79,6 @@ lodash "^4.17.13" source-map "^0.5.0" -"@babel/generator@^7.11.0", "@babel/generator@^7.4.0": - version "7.11.0" - resolved "https://registry.npmjs.org/@babel/generator/-/generator-7.11.0.tgz#4b90c78d8c12825024568cbe83ee6c9af193585c" - integrity sha512-fEm3Uzw7Mc9Xi//qU20cBKatTfs2aOtKqmvy/Vm7RkJEGFQ4xc9myCfbXxqK//ZS8MR/ciOHw6meGASJuKmDfQ== - dependencies: - "@babel/types" "^7.11.0" - jsesc "^2.5.1" - source-map "^0.5.0" - "@babel/generator@^7.9.0": version "7.9.4" resolved "https://registry.npmjs.org/@babel/generator/-/generator-7.9.4.tgz#12441e90c3b3c4159cdecf312075bf1a8ce2dbce" @@ -363,13 +354,6 @@ dependencies: "@babel/types" "^7.10.4" -"@babel/helper-split-export-declaration@^7.11.0": - version "7.11.0" - resolved "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.11.0.tgz#f8a491244acf6a676158ac42072911ba83ad099f" - integrity sha512-74Vejvp6mHkGE+m+k5vHY93FX2cAtrw1zXrZXRlG4l410Nm9PxfEiVTn1PjDPV5SnmieiueY4AFg2xqhNFuuZg== - dependencies: - "@babel/types" "^7.11.0" - "@babel/helper-split-export-declaration@^7.8.3": version "7.8.3" resolved "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.8.3.tgz#31a9f30070f91368a7182cf05f831781065fc7a9" @@ -443,11 +427,6 @@ resolved "https://registry.npmjs.org/@babel/parser/-/parser-7.10.4.tgz#9eedf27e1998d87739fb5028a5120557c06a1a64" integrity sha512-8jHII4hf+YVDsskTF6WuMB3X4Eh+PsUkC2ljq22so5rHvH+T8BzyL94VOdyFLNR8tBSVXOTbNHOKpR4TfRxVtA== -"@babel/parser@^7.11.0", "@babel/parser@^7.4.3": - version "7.11.0" - resolved "https://registry.npmjs.org/@babel/parser/-/parser-7.11.0.tgz#a9d7e11aead25d3b422d17b2c6502c8dddef6a5d" - integrity sha512-qvRvi4oI8xii8NllyEc4MDJjuZiNaRzyb7Y7lup1NqJV8TZHF4O27CcP+72WPn/k1zkgJ6WJfnIbk4jTsVAZHw== - "@babel/parser@^7.7.5", "@babel/parser@^7.8.6", "@babel/parser@^7.9.0": version "7.9.4" resolved "https://registry.npmjs.org/@babel/parser/-/parser-7.9.4.tgz#68a35e6b0319bbc014465be43828300113f2f2e8" @@ -983,7 +962,7 @@ dependencies: regenerator-runtime "^0.13.4" -"@babel/template@^7.10.4", "@babel/template@^7.4.0": +"@babel/template@^7.10.4": version "7.10.4" resolved "https://registry.npmjs.org/@babel/template/-/template-7.10.4.tgz#3251996c4200ebc71d1a8fc405fba940f36ba278" integrity sha512-ZCjD27cGJFUB6nmCB1Enki3r+L5kJveX9pq1SvAUKoICy6CZ9yD8xO086YXdYhvNjBdnekm4ZnaP5yC8Cs/1tA== @@ -1016,21 +995,6 @@ globals "^11.1.0" lodash "^4.17.13" -"@babel/traverse@^7.4.3": - version "7.11.0" - resolved "https://registry.npmjs.org/@babel/traverse/-/traverse-7.11.0.tgz#9b996ce1b98f53f7c3e4175115605d56ed07dd24" - integrity sha512-ZB2V+LskoWKNpMq6E5UUCrjtDUh5IOTAyIl0dTjIEoXum/iKWkoIEKIRDnUucO6f+2FzNkE0oD4RLKoPIufDtg== - dependencies: - "@babel/code-frame" "^7.10.4" - "@babel/generator" "^7.11.0" - "@babel/helper-function-name" "^7.10.4" - "@babel/helper-split-export-declaration" "^7.11.0" - "@babel/parser" "^7.11.0" - "@babel/types" "^7.11.0" - debug "^4.1.0" - globals "^11.1.0" - lodash "^4.17.19" - "@babel/traverse@^7.7.4", "@babel/traverse@^7.8.6", "@babel/traverse@^7.9.0": version "7.9.0" resolved "https://registry.npmjs.org/@babel/traverse/-/traverse-7.9.0.tgz#d3882c2830e513f4fe4cec9fe76ea1cc78747892" @@ -1055,15 +1019,6 @@ lodash "^4.17.13" to-fast-properties "^2.0.0" -"@babel/types@^7.11.0", "@babel/types@^7.4.0": - version "7.11.0" - resolved "https://registry.npmjs.org/@babel/types/-/types-7.11.0.tgz#2ae6bf1ba9ae8c3c43824e5861269871b206e90d" - integrity sha512-O53yME4ZZI0jO1EVGtF1ePGl0LHirG4P1ibcD80XyzZcKhcMFeCXmh4Xb1ifGBIV233Qg12x4rBfQgA+tmOukA== - dependencies: - "@babel/helper-validator-identifier" "^7.10.4" - lodash "^4.17.19" - to-fast-properties "^2.0.0" - "@babel/types@^7.4.4": version "7.9.5" resolved "https://registry.npmjs.org/@babel/types/-/types-7.9.5.tgz#89231f82915a8a566a703b3b20133f73da6b9444" @@ -3305,13 +3260,6 @@ append-buffer@^1.0.2: dependencies: buffer-equal "^1.0.0" -append-transform@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/append-transform/-/append-transform-1.0.0.tgz#046a52ae582a228bd72f58acfbe2967c678759ab" - integrity sha512-P009oYkeHyU742iSZJzZZywj4QRJdnTWffaKuJQLablCZ1uz6/cW4yaRgcDaoQ+uwOxxnt0gRUcwfsNP2ri0gw== - dependencies: - default-require-extensions "^2.0.0" - append-transform@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/append-transform/-/append-transform-2.0.0.tgz#99d9d29c7b38391e6f428d28ce136551f0b77e12" @@ -4960,7 +4908,7 @@ compare-semver@^1.0.0: dependencies: semver "^5.0.1" -compare-versions@^3.4.0, compare-versions@^3.6.0: +compare-versions@^3.6.0: version "3.6.0" resolved "https://registry.npmjs.org/compare-versions/-/compare-versions-3.6.0.tgz#1a5689913685e5a87637b8d3ffca75514ec41d62" integrity sha512-W6Af2Iw1z4CB7q4uU4hv646dW9GQuBM+YpC0UvUCWSD8w90SJjp+ujJuXaEMtAXBtSqGfMPuFOVn4/+FlaqfBA== @@ -5683,13 +5631,6 @@ default-compare@^1.0.0: dependencies: kind-of "^5.0.2" -default-require-extensions@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-2.0.0.tgz#f5f8fbb18a7d6d50b21f641f649ebb522cfe24f7" - integrity sha1-9fj7sYp9bVCyH2QfZJ67Uiz+JPc= - dependencies: - strip-bom "^3.0.0" - default-require-extensions@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-3.0.0.tgz#e03f93aac9b2b6443fc52e5e4a37b3ad9ad8df96" @@ -6865,14 +6806,6 @@ file-uri-to-path@1.0.0: resolved "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== -fileset@^2.0.3: - version "2.0.3" - resolved "https://registry.npmjs.org/fileset/-/fileset-2.0.3.tgz#8e7548a96d3cc2327ee5e674168723a333bba2a0" - integrity sha1-jnVIqW08wjJ+5eZ0FocjozO7oqA= - dependencies: - glob "^7.0.3" - minimatch "^3.0.3" - filesize@^3.1.3: version "3.6.1" resolved "https://registry.npmjs.org/filesize/-/filesize-3.6.1.tgz#090bb3ee01b6f801a8a8be99d31710b3422bb317" @@ -9130,25 +9063,6 @@ isstream@~0.1.2: resolved "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo= -istanbul-api@^2.1.6: - version "2.1.7" - resolved "https://registry.npmjs.org/istanbul-api/-/istanbul-api-2.1.7.tgz#82786b79f3b93d481349c7aa1e2c2b4eeb48c8a8" - integrity sha512-LYTOa2UrYFyJ/aSczZi/6lBykVMjCCvUmT64gOe+jPZFy4w6FYfPGqFT2IiQ2BxVHHDOvCD7qrIXb0EOh4uGWw== - dependencies: - async "^2.6.2" - compare-versions "^3.4.0" - fileset "^2.0.3" - istanbul-lib-coverage "^2.0.5" - istanbul-lib-hook "^2.0.7" - istanbul-lib-instrument "^3.3.0" - istanbul-lib-report "^2.0.8" - istanbul-lib-source-maps "^3.0.6" - istanbul-reports "^2.2.5" - js-yaml "^3.13.1" - make-dir "^2.1.0" - minimatch "^3.0.4" - once "^1.4.0" - istanbul-instrumenter-loader@3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/istanbul-instrumenter-loader/-/istanbul-instrumenter-loader-3.0.1.tgz#9957bd59252b373fae5c52b7b5188e6fde2a0949" @@ -9174,13 +9088,6 @@ istanbul-lib-coverage@^3.0.0, istanbul-lib-coverage@^3.0.0-alpha.1: resolved "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.0.0.tgz#f5944a37c70b550b02a78a5c3b2055b280cec8ec" integrity sha512-UiUIqxMgRDET6eR+o5HbfRYP1l0hqkWOs7vNxC/mggutCMUIhWMm8gAHb8tHlyfD3/l6rlgNA5cKdDzEAf6hEg== -istanbul-lib-hook@^2.0.7: - version "2.0.7" - resolved "https://registry.npmjs.org/istanbul-lib-hook/-/istanbul-lib-hook-2.0.7.tgz#c95695f383d4f8f60df1f04252a9550e15b5b133" - integrity sha512-vrRztU9VRRFDyC+aklfLoeXyNdTfga2EI3udDGn4cZ6fpSXpHLV9X6CHvfoMCPtggg8zvDDmC4b9xfu0z6/llA== - dependencies: - append-transform "^1.0.0" - istanbul-lib-hook@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/istanbul-lib-hook/-/istanbul-lib-hook-3.0.0.tgz#8f84c9434888cc6b1d0a9d7092a76d239ebf0cc6" @@ -9201,19 +9108,6 @@ istanbul-lib-instrument@^1.7.3: istanbul-lib-coverage "^1.2.1" semver "^5.3.0" -istanbul-lib-instrument@^3.3.0: - version "3.3.0" - resolved "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-3.3.0.tgz#a5f63d91f0bbc0c3e479ef4c5de027335ec6d630" - integrity sha512-5nnIN4vo5xQZHdXno/YDXJ0G+I3dAm4XgzfSVTPLQpj/zAV2dV6Juy0yaf10/zrJOJeHoN3fraFe+XRq2bFVZA== - dependencies: - "@babel/generator" "^7.4.0" - "@babel/parser" "^7.4.3" - "@babel/template" "^7.4.0" - "@babel/traverse" "^7.4.3" - "@babel/types" "^7.4.0" - istanbul-lib-coverage "^2.0.5" - semver "^6.0.0" - istanbul-lib-instrument@^4.0.0: version "4.0.1" resolved "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-4.0.1.tgz#61f13ac2c96cfefb076fe7131156cc05907874e6" @@ -9240,15 +9134,6 @@ istanbul-lib-processinfo@^2.0.2: rimraf "^3.0.0" uuid "^3.3.3" -istanbul-lib-report@^2.0.8: - version "2.0.8" - resolved "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-2.0.8.tgz#5a8113cd746d43c4889eba36ab10e7d50c9b4f33" - integrity sha512-fHBeG573EIihhAblwgxrSenp0Dby6tJMFR/HvlerBsrCTD5bkUuoNtn3gVh29ZCS824cGGBPn7Sg7cNk+2xUsQ== - dependencies: - istanbul-lib-coverage "^2.0.5" - make-dir "^2.1.0" - supports-color "^6.1.0" - istanbul-lib-report@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz#7518fe52ea44de372f460a76b5ecda9ffb73d8a6" @@ -9278,13 +9163,6 @@ istanbul-lib-source-maps@^4.0.0: istanbul-lib-coverage "^3.0.0" source-map "^0.6.1" -istanbul-reports@^2.2.5: - version "2.2.7" - resolved "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-2.2.7.tgz#5d939f6237d7b48393cc0959eab40cd4fd056931" - integrity sha512-uu1F/L1o5Y6LzPVSVZXNOoD/KXpJue9aeLRd0sM9uMXfZvzomB0WxVamWb5ue8kA2vVWEmW7EG+A5n3f1kqHKg== - dependencies: - html-escaper "^2.0.0" - istanbul-reports@^3.0.0, istanbul-reports@^3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.0.2.tgz#d593210e5000683750cb09fc0644e4b6e27fd53b" @@ -9594,12 +9472,15 @@ karma-cli@2.0.0: dependencies: resolve "^1.3.3" -karma-coverage-istanbul-reporter@2.1.1: - version "2.1.1" - resolved "https://registry.npmjs.org/karma-coverage-istanbul-reporter/-/karma-coverage-istanbul-reporter-2.1.1.tgz#37a775fbfbb3cbe98cebf19605c94c6277c3b88a" - integrity sha512-CH8lTi8+kKXGvrhy94+EkEMldLCiUA0xMOiL31vvli9qK0T+qcXJAwWBRVJWnVWxYkTmyWar8lPz63dxX6/z1A== +karma-coverage-istanbul-reporter@3.0.3: + version "3.0.3" + resolved "https://registry.npmjs.org/karma-coverage-istanbul-reporter/-/karma-coverage-istanbul-reporter-3.0.3.tgz#f3b5303553aadc8e681d40d360dfdc19bc7e9fe9" + integrity sha512-wE4VFhG/QZv2Y4CdAYWDbMmcAHeS926ZIji4z+FkB2aF/EposRb6DP6G5ncT/wXhqUfAb/d7kZrNKPonbvsATw== dependencies: - istanbul-api "^2.1.6" + istanbul-lib-coverage "^3.0.0" + istanbul-lib-report "^3.0.0" + istanbul-lib-source-maps "^3.0.6" + istanbul-reports "^3.0.2" minimatch "^3.0.4" karma-firefox-launcher@1.3.0: @@ -10276,7 +10157,7 @@ lodash@4.17.15, lodash@^4.0.0, lodash@^4.16.6, lodash@^4.17.10, lodash@^4.17.11, resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A== -lodash@4.17.19, lodash@^4.17.19: +lodash@4.17.19: version "4.17.19" resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz#e48ddedbe30b3321783c5b4301fbd353bc1e4a4b" integrity sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ== @@ -10730,7 +10611,7 @@ minimalistic-crypto-utils@^1.0.0, minimalistic-crypto-utils@^1.0.1: resolved "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz#f6c00c1c0b082246e5c4d99dfb8c7c083b2b582a" integrity sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo= -minimatch@3.0.4, minimatch@^3.0.0, minimatch@^3.0.3, minimatch@^3.0.4: +minimatch@3.0.4, minimatch@^3.0.0, minimatch@^3.0.4: version "3.0.4" resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== From 9f1e135520e7028796ec668e7cd3a33b1ab5a267 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Wed, 5 Aug 2020 12:42:53 -0400 Subject: [PATCH 41/43] Address feedback --- packages/firestore/exp/src/api/database.ts | 2 +- packages/firestore/src/local/local_store.ts | 150 ++++++++---------- packages/firestore/src/remote/serializer.ts | 2 + .../integration/api_internal/bundle.test.ts | 3 +- .../test/unit/local/local_store.test.ts | 19 ++- .../test/unit/specs/bundle_spec.test.ts | 12 +- .../firestore/test/unit/specs/spec_builder.ts | 46 ++++-- .../test/unit/specs/spec_test_runner.ts | 7 +- .../firestore/test/unit/util/bundle_data.ts | 21 ++- 9 files changed, 139 insertions(+), 123 deletions(-) diff --git a/packages/firestore/exp/src/api/database.ts b/packages/firestore/exp/src/api/database.ts index 20e9d489a58..9e7598099a2 100644 --- a/packages/firestore/exp/src/api/database.ts +++ b/packages/firestore/exp/src/api/database.ts @@ -323,5 +323,5 @@ export async function namedQuery( return null; } - return new Query(firestoreImpl, null, namedQuery.query, namedQuery.readTime); + return new Query(firestoreImpl, null, namedQuery.query); } diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 6cc75c1491e..2925bd69527 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -237,20 +237,16 @@ export interface LocalStore { * they don't get GC'd. A target must be allocated in the local store before * the store can be used to manage its view. * - * @param {SnapshotVersion} readTime The snapshot version the allocated target + * @param {SnapshotVersion} readFrom The snapshot version the allocated target * should resume from, unless the existing `TargetData` has an even newer * version. - * @param {PersistenceTransaction} txn The transaction to use if allocation is - * part of an existing transaction. A new transaction is created if this is - * absent. * * Allocating an already allocated `Target` will return the existing `TargetData` * for that `Target`. */ allocateTarget( target: Target, - readTime?: SnapshotVersion, - txn?: PersistenceTransaction + readFrom?: SnapshotVersion ): Promise; /** @@ -902,84 +898,54 @@ class LocalStoreImpl implements LocalStore { }); } - _allocateOperation( - txn: PersistenceTransaction, - target: Target, - readTime?: SnapshotVersion - ): PersistencePromise { - let targetData: TargetData; - return this.targetCache - .getTargetData(txn, target) - .next((cached: TargetData | null) => { - if (cached) { - // This target has been listened to previously, so reuse the - // previous targetID if no `readTime` is provided. - // If `readTime` is provided, the cached target data needs to be - // compared and updated if `readTime` is newer. - // TODO(mcg): freshen last accessed date? - if (readTime && cached.snapshotVersion.compareTo(readTime) < 0) { - targetData = cached.withReadTime(readTime); - return this.targetCache - .updateTargetData(txn, targetData) - .next(() => targetData); - } - targetData = cached; - return PersistencePromise.resolve(targetData); - } else { - return this.targetCache.allocateTargetId(txn).next(targetId => { - targetData = new TargetData( - target, - targetId, - TargetPurpose.Listen, - txn.currentSequenceNumber, - readTime - ); - return this.targetCache - .addTargetData(txn, targetData) - .next(() => targetData); + allocateTarget(target: Target): Promise { + return this.persistence + .runTransaction('Allocate target', 'readwrite', txn => { + let targetData: TargetData; + return this.targetCache + .getTargetData(txn, target) + .next((cached: TargetData | null) => { + if (cached) { + // TODO(mcg): freshen last accessed date? + targetData = cached; + return PersistencePromise.resolve(targetData); + } else { + return this.targetCache.allocateTargetId(txn).next(targetId => { + targetData = new TargetData( + target, + targetId, + TargetPurpose.Listen, + txn.currentSequenceNumber + ); + return this.targetCache + .addTargetData(txn, targetData) + .next(() => targetData); + }); + } }); + }) + .then(targetData => { + // If Multi-Tab is enabled, the existing target data may be newer than + // the in-memory data + const cachedTargetData = this.targetDataByTarget.get( + targetData.targetId + ); + if ( + cachedTargetData === null || + targetData.snapshotVersion.compareTo( + cachedTargetData.snapshotVersion + ) > 0 + ) { + this.targetDataByTarget = this.targetDataByTarget.insert( + targetData.targetId, + targetData + ); + this.targetIdByTarget.set(target, targetData.targetId); } + return targetData; }); } - allocateTarget( - target: Target, - readTime?: SnapshotVersion, - txn?: PersistenceTransaction - ): Promise { - let allocateOperation: Promise; - if (txn) { - allocateOperation = this._allocateOperation( - txn, - target, - readTime - ).toPromise(); - } else { - allocateOperation = this.persistence.runTransaction( - 'Allocate target', - 'readwrite', - txn => this._allocateOperation(txn, target, readTime) - ); - } - return allocateOperation.then(targetData => { - // If Multi-Tab is enabled, the existing target data may be newer than - // the in-memory data - const cachedTargetData = this.targetDataByTarget.get(targetData.targetId); - if ( - cachedTargetData === null || - targetData.snapshotVersion.compareTo(cachedTargetData.snapshotVersion) > - 0 - ) { - this.targetDataByTarget = this.targetDataByTarget.insert( - targetData.targetId, - targetData - ); - this.targetIdByTarget.set(target, targetData.targetId); - } - return targetData; - }); - } - getTargetData( transaction: PersistenceTransaction, target: Target @@ -1411,22 +1377,34 @@ export function getNamedQuery( /** * Saves the given `NamedQuery` to local persistence. */ -export function saveNamedQuery( +export async function saveNamedQuery( localStore: LocalStore, query: bundleProto.NamedQuery ): Promise { + // Allocate a target for the named query such that it can be resumed + // from associated read time if users use it to listen. It will be + // removed through GC as an ordinary query. + const allocated = await localStore.allocateTarget( + queryToTarget(fromBundledQuery(query.bundledQuery!)) + ); const localStoreImpl = debugCast(localStore, LocalStoreImpl); return localStoreImpl.persistence.runTransaction( 'Save named query', 'readwrite', transaction => { - // eslint-disable-next-line @typescript-eslint/no-floating-promises - localStore.allocateTarget( - queryToTarget(fromBundledQuery(query.bundledQuery!)), - fromVersion(query.readTime!), - transaction + // Update allocated target's read time, if the bundle's read time is newer. + let updateReadTime = PersistencePromise.resolve(); + const readTime = fromVersion(query.readTime!); + if (allocated.snapshotVersion.compareTo(readTime) < 0) { + const newTargetData = allocated.withReadTime(readTime); + updateReadTime = localStoreImpl.targetCache.updateTargetData( + transaction, + newTargetData + ); + } + return updateReadTime.next(() => + localStoreImpl.bundleCache.saveNamedQuery(transaction, query) ); - return localStoreImpl.bundleCache.saveNamedQuery(transaction, query); } ); } diff --git a/packages/firestore/src/remote/serializer.ts b/packages/firestore/src/remote/serializer.ts index 0e9da4b6111..5b708215258 100644 --- a/packages/firestore/src/remote/serializer.ts +++ b/packages/firestore/src/remote/serializer.ts @@ -973,6 +973,8 @@ export function toTarget( result.targetId = targetData.targetId; + // TODO(wuandy): We should consider remove resume token since we can always + // use readtime. if (targetData.resumeToken.approximateByteSize() > 0) { result.resumeToken = toBytes(serializer, targetData.resumeToken); } else if (targetData.snapshotVersion.compareTo(SnapshotVersion.min()) > 0) { diff --git a/packages/firestore/test/integration/api_internal/bundle.test.ts b/packages/firestore/test/integration/api_internal/bundle.test.ts index 74e27189e8c..6581404c6d5 100644 --- a/packages/firestore/test/integration/api_internal/bundle.test.ts +++ b/packages/firestore/test/integration/api_internal/bundle.test.ts @@ -80,8 +80,7 @@ apiDescribe('Bundles', (persistence: boolean) => { (collectionReference('coll-1') .orderBy('bar', 'desc') // eslint-disable-next-line @typescript-eslint/no-explicit-any - .limit(1) as any)._query, - 'LAST' + .limitToLast(1) as any)._query ); builder.addDocumentMetadata(a, { seconds: 1000, nanos: 9999 }, true); diff --git a/packages/firestore/test/unit/local/local_store.test.ts b/packages/firestore/test/unit/local/local_store.test.ts index d594c4d3de6..9e3871a9af6 100644 --- a/packages/firestore/test/unit/local/local_store.test.ts +++ b/packages/firestore/test/unit/local/local_store.test.ts @@ -447,7 +447,7 @@ class LocalStoreTester { toHaveNamedQuery(namedQuery: NamedQuery): LocalStoreTester { this.promiseChain = this.promiseChain.then(() => { return getNamedQuery(this.localStore, namedQuery.name).then(actual => { - expect(!!actual).to.be.true; + expect(actual).to.exist; expect(actual!.name).to.equal(namedQuery.name); expect(namedQuery.readTime.isEqual(actual!.readTime)).to.be.true; expect(queryEquals(actual!.query, namedQuery.query)).to.be.true; @@ -1690,9 +1690,16 @@ function genericLocalStoreTests( it('handles saving and loading named queries', async () => { return expectLocalStore() - .after(namedQuery('test', query('coll'), 'FIRST', SnapshotVersion.min())) + .after( + namedQuery( + 'testQueryName', + query('coll'), + /* limitType */ 'FIRST', + SnapshotVersion.min() + ) + ) .toHaveNamedQuery({ - name: 'test', + name: 'testQueryName', query: query('coll'), readTime: SnapshotVersion.min() }) @@ -1704,14 +1711,14 @@ function genericLocalStoreTests( return expectLocalStore() .after( namedQuery( - 'test', + 'testQueryName', queryWithLimit(query('coll', orderBy('sort')), 5, LimitType.First), - 'LAST', + /* limitType */ 'LAST', SnapshotVersion.fromTimestamp(now) ) ) .toHaveNamedQuery({ - name: 'test', + name: 'testQueryName', query: queryWithLimit( query('coll', orderBy('sort')), 5, diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts index 5b0ced3ae9b..d87c6980f06 100644 --- a/packages/firestore/test/unit/specs/bundle_spec.test.ts +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -62,8 +62,7 @@ function bundleWithDocumentAndQuery( builder.addNamedQuery( testQuery.name, toVersion(JSON_SERIALIZER, version(testQuery.readTime)), - testQuery.query, - testQuery.limitType + testQuery.query ); } @@ -308,7 +307,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { return spec() .loadBundle(bundleString1) - .userListens(query1, 400) + .userListens(query1, /* resumeFrom */ '', 400) .expectEvents(query1, { added: [doc('collection/a', 500, { value: 'b' })], fromCache: true @@ -349,7 +348,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { .loadBundle(bundleString1) // Read named query from loaded bundle by primary. .client(1) - .userListens(query1, 400) + .userListens(query1, /* resumeFrom */ '', 400) .expectEvents(query1, { added: [doc('collection/a', 500, { value: 'b' })], fromCache: true @@ -360,10 +359,11 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { modified: [doc('collection/a', 550, { value: 'c' })], fromCache: true }) - .userUnlistens(query1) // Read named query from loaded bundle by secondary. .client(0) - .userListens(query2, 560) + .expectListen(query1, '', 400) + .expectActiveTargets({ query: query1, readTime: 400 }) + .userListens(query2, /* resumeFrom */ '', 560) .expectEvents(query2, { added: [doc('collection/a', 550, { value: 'c' })], fromCache: true diff --git a/packages/firestore/test/unit/specs/spec_builder.ts b/packages/firestore/test/unit/specs/spec_builder.ts index 79b69ec2802..87b514ee0b1 100644 --- a/packages/firestore/test/unit/specs/spec_builder.ts +++ b/packages/firestore/test/unit/specs/spec_builder.ts @@ -73,7 +73,8 @@ export interface LimboMap { export interface ActiveTargetSpec { queries: SpecQuery[]; - resumeToken: string | TestSnapshotVersion; + resumeToken?: string; + readTime?: TestSnapshotVersion; } export interface ActiveTargetMap { @@ -253,7 +254,11 @@ export class SpecBuilder { return this; } - userListens(query: Query, resumeFrom?: string | TestSnapshotVersion): this { + userListens( + query: Query, + resumeFrom?: string, + readTime?: TestSnapshotVersion + ): this { this.nextStep(); const target = queryToTarget(query); @@ -272,7 +277,7 @@ export class SpecBuilder { } this.queryMapping.set(target, targetId); - this.addQueryToActiveTargets(targetId, query, resumeFrom); + this.addQueryToActiveTargets(targetId, query, resumeFrom, readTime); this.currentStep = { userListen: { targetId, query: SpecBuilder.queryToSpec(query) }, expectedState: { activeTargets: { ...this.activeTargets } } @@ -493,13 +498,22 @@ export class SpecBuilder { /** Overrides the currently expected set of active targets. */ expectActiveTargets( - ...targets: Array<{ query: Query; resumeToken?: string }> + ...targets: Array<{ + query: Query; + resumeToken?: string; + readTime?: TestSnapshotVersion; + }> ): this { this.assertStep('Active target expectation requires previous step'); const currentStep = this.currentStep!; this.clientState.activeTargets = {}; - targets.forEach(({ query, resumeToken }) => { - this.addQueryToActiveTargets(this.getTargetId(query), query, resumeToken); + targets.forEach(({ query, resumeToken, readTime }) => { + this.addQueryToActiveTargets( + this.getTargetId(query), + query, + resumeToken, + readTime + ); }); currentStep.expectedState = currentStep.expectedState || {}; currentStep.expectedState.activeTargets = { ...this.activeTargets }; @@ -860,14 +874,18 @@ export class SpecBuilder { } /** Registers a query that is active in another tab. */ - expectListen(query: Query, resumeToken?: string): this { + expectListen( + query: Query, + resumeToken?: string, + readTime?: TestSnapshotVersion + ): this { this.assertStep('Expectations require previous step'); const target = queryToTarget(query); const targetId = this.queryIdGenerator.cachedId(target); this.queryMapping.set(target, targetId); - this.addQueryToActiveTargets(targetId, query, resumeToken); + this.addQueryToActiveTargets(targetId, query, resumeToken, readTime); const currentStep = this.currentStep!; currentStep.expectedState = currentStep.expectedState || {}; @@ -1029,7 +1047,8 @@ export class SpecBuilder { private addQueryToActiveTargets( targetId: number, query: Query, - resumeToken?: string | TestSnapshotVersion + resumeToken?: string, + readTime?: TestSnapshotVersion ): void { if (this.activeTargets[targetId]) { const activeQueries = this.activeTargets[targetId].queries; @@ -1041,18 +1060,21 @@ export class SpecBuilder { // `query` is not added yet. this.activeTargets[targetId] = { queries: [SpecBuilder.queryToSpec(query), ...activeQueries], - resumeToken: resumeToken || '' + resumeToken: resumeToken || '', + readTime }; } else { this.activeTargets[targetId] = { queries: activeQueries, - resumeToken: resumeToken || '' + resumeToken: resumeToken || '', + readTime }; } } else { this.activeTargets[targetId] = { queries: [SpecBuilder.queryToSpec(query)], - resumeToken: resumeToken || '' + resumeToken: resumeToken || '', + readTime }; } } diff --git a/packages/firestore/test/unit/specs/spec_test_runner.ts b/packages/firestore/test/unit/specs/spec_test_runner.ts index e58683db68a..46bdacaf8fa 100644 --- a/packages/firestore/test/unit/specs/spec_test_runner.ts +++ b/packages/firestore/test/unit/specs/spec_test_runner.ts @@ -984,16 +984,13 @@ abstract class TestRunner { TargetPurpose.Listen, ARBITRARY_SEQUENCE_NUMBER ); - if (typeof expected.resumeToken === 'string') { + if (expected.resumeToken && expected.resumeToken !== '') { targetData = targetData.withResumeToken( byteStringFromString(expected.resumeToken), SnapshotVersion.min() ); } else { - targetData = targetData.withResumeToken( - byteStringFromString(''), - version(expected.resumeToken) - ); + targetData = targetData.withReadTime(version(expected.readTime!)); } const expectedTarget = toTarget(this.serializer, targetData); expect(actualTarget.query).to.deep.equal(expectedTarget.query); diff --git a/packages/firestore/test/unit/util/bundle_data.ts b/packages/firestore/test/unit/util/bundle_data.ts index 987fc7aacf2..386256e0e3b 100644 --- a/packages/firestore/test/unit/util/bundle_data.ts +++ b/packages/firestore/test/unit/util/bundle_data.ts @@ -16,7 +16,7 @@ */ import { BundleElement, - LimitType + LimitType as BundleLimitType } from '../../../src/protos/firestore_bundle_proto'; import { DatabaseId } from '../../../src/core/database_info'; import * as api from '../../../src/protos/firestore_proto_api'; @@ -31,7 +31,12 @@ import { newSerializer, newTextEncoder } from '../../../src/platform/serializer'; -import { Query, queryToTarget } from '../../../src/core/query'; +import { + LimitType, + Query, + queryToTarget, + queryWithLimit +} from '../../../src/core/query'; export const encoder = newTextEncoder(); @@ -82,9 +87,15 @@ export class TestBundleBuilder { addNamedQuery( name: string, readTime: api.Timestamp, - query: Query, - limitType?: LimitType + query: Query ): TestBundleBuilder { + let bundledLimitType: BundleLimitType | undefined = !!query.limit + ? 'FIRST' + : undefined; + if (query.limitType === LimitType.Last) { + query = queryWithLimit(query, query.limit!, LimitType.First); + bundledLimitType = 'LAST'; + } const queryTarget = toQueryTarget(this.serializer, queryToTarget(query)); this.elements.push({ namedQuery: { @@ -93,7 +104,7 @@ export class TestBundleBuilder { bundledQuery: { parent: queryTarget.parent, structuredQuery: queryTarget.structuredQuery, - limitType + limitType: bundledLimitType } } }); From 5eb05d9c98d35dfde116574368ca660949e95ce2 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Wed, 5 Aug 2020 12:50:03 -0400 Subject: [PATCH 42/43] Remove readTime from allocateTarget --- packages/firestore/src/local/local_store.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 2925bd69527..c8b747412d6 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -237,17 +237,10 @@ export interface LocalStore { * they don't get GC'd. A target must be allocated in the local store before * the store can be used to manage its view. * - * @param {SnapshotVersion} readFrom The snapshot version the allocated target - * should resume from, unless the existing `TargetData` has an even newer - * version. - * * Allocating an already allocated `Target` will return the existing `TargetData` * for that `Target`. */ - allocateTarget( - target: Target, - readFrom?: SnapshotVersion - ): Promise; + allocateTarget(target: Target): Promise; /** * Returns the TargetData as seen by the LocalStore, including updates that may From e057fc14b2c2b9e91592fcafa763ea6bb92d73ca Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Wed, 5 Aug 2020 16:12:57 -0400 Subject: [PATCH 43/43] drop resume token when updating read time. Also better spec tests. --- packages/firestore/src/local/local_store.ts | 12 +++-- packages/firestore/src/local/target_data.ts | 13 ----- packages/firestore/src/remote/serializer.ts | 5 +- .../test/unit/specs/bundle_spec.test.ts | 8 +-- .../unit/specs/existence_filter_spec.test.ts | 2 +- .../test/unit/specs/limbo_spec.test.ts | 8 +-- .../test/unit/specs/limit_spec.test.ts | 14 ++--- .../test/unit/specs/listen_spec.test.ts | 54 +++++++++---------- .../test/unit/specs/orderby_spec.test.ts | 2 +- .../test/unit/specs/perf_spec.test.ts | 4 +- .../test/unit/specs/persistence_spec.test.ts | 4 +- .../test/unit/specs/recovery_spec.test.ts | 10 ++-- .../test/unit/specs/resume_token_spec.test.ts | 2 +- .../firestore/test/unit/specs/spec_builder.ts | 20 ++++--- .../test/unit/specs/spec_test_runner.ts | 5 +- .../test/unit/specs/write_spec.test.ts | 2 +- 16 files changed, 86 insertions(+), 79 deletions(-) diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index c8b747412d6..cfb6ee8370e 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -78,6 +78,7 @@ import { BundleConverter, BundledDocuments, NamedQuery } from '../core/bundle'; import { BundleCache } from './bundle_cache'; import { fromVersion, JsonProtoSerializer } from '../remote/serializer'; import { fromBundledQuery } from './local_serializer'; +import { ByteString } from '../util/byte_string'; const LOG_TAG = 'LocalStore'; @@ -1375,8 +1376,10 @@ export async function saveNamedQuery( query: bundleProto.NamedQuery ): Promise { // Allocate a target for the named query such that it can be resumed - // from associated read time if users use it to listen. It will be - // removed through GC as an ordinary query. + // from associated read time if users use it to listen. + // NOTE: this also means if no corresponding target exists, the new target + // will remain active and will not get collected, unless users happen to + // unlisten the query somehow. const allocated = await localStore.allocateTarget( queryToTarget(fromBundledQuery(query.bundledQuery!)) ); @@ -1389,7 +1392,10 @@ export async function saveNamedQuery( let updateReadTime = PersistencePromise.resolve(); const readTime = fromVersion(query.readTime!); if (allocated.snapshotVersion.compareTo(readTime) < 0) { - const newTargetData = allocated.withReadTime(readTime); + const newTargetData = allocated.withResumeToken( + ByteString.EMPTY_BYTE_STRING, + readTime + ); updateReadTime = localStoreImpl.targetCache.updateTargetData( transaction, newTargetData diff --git a/packages/firestore/src/local/target_data.ts b/packages/firestore/src/local/target_data.ts index fad5aeec763..95237e574a7 100644 --- a/packages/firestore/src/local/target_data.ts +++ b/packages/firestore/src/local/target_data.ts @@ -69,19 +69,6 @@ export class TargetData { readonly resumeToken: ByteString = ByteString.EMPTY_BYTE_STRING ) {} - /** Creates a new target data instance with an updated read time. */ - withReadTime(version: SnapshotVersion): TargetData { - return new TargetData( - this.target, - this.targetId, - this.purpose, - this.sequenceNumber, - version, - this.lastLimboFreeSnapshotVersion, - this.resumeToken - ); - } - /** Creates a new target data instance with an updated sequence number. */ withSequenceNumber(sequenceNumber: number): TargetData { return new TargetData( diff --git a/packages/firestore/src/remote/serializer.ts b/packages/firestore/src/remote/serializer.ts index 5b708215258..77499d4e406 100644 --- a/packages/firestore/src/remote/serializer.ts +++ b/packages/firestore/src/remote/serializer.ts @@ -973,11 +973,12 @@ export function toTarget( result.targetId = targetData.targetId; - // TODO(wuandy): We should consider remove resume token since we can always - // use readtime. if (targetData.resumeToken.approximateByteSize() > 0) { result.resumeToken = toBytes(serializer, targetData.resumeToken); } else if (targetData.snapshotVersion.compareTo(SnapshotVersion.min()) > 0) { + // TODO(wuandy): Consider removing above check because it is most likely true. + // Right now, many tests depend on this behaviour though (leaving min() out + // of serialization). result.readTime = toTimestamp( serializer, targetData.snapshotVersion.toTimestamp() diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts index d87c6980f06..85399afd946 100644 --- a/packages/firestore/test/unit/specs/bundle_spec.test.ts +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -307,7 +307,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { return spec() .loadBundle(bundleString1) - .userListens(query1, /* resumeFrom */ '', 400) + .userListens(query1, { readTime: 400 }) .expectEvents(query1, { added: [doc('collection/a', 500, { value: 'b' })], fromCache: true @@ -348,7 +348,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { .loadBundle(bundleString1) // Read named query from loaded bundle by primary. .client(1) - .userListens(query1, /* resumeFrom */ '', 400) + .userListens(query1, { readTime: 400 }) .expectEvents(query1, { added: [doc('collection/a', 500, { value: 'b' })], fromCache: true @@ -361,9 +361,9 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { }) // Read named query from loaded bundle by secondary. .client(0) - .expectListen(query1, '', 400) + .expectListen(query1, { readTime: 400 }) .expectActiveTargets({ query: query1, readTime: 400 }) - .userListens(query2, /* resumeFrom */ '', 560) + .userListens(query2, { readTime: 560 }) .expectEvents(query2, { added: [doc('collection/a', 550, { value: 'c' })], fromCache: true diff --git a/packages/firestore/test/unit/specs/existence_filter_spec.test.ts b/packages/firestore/test/unit/specs/existence_filter_spec.test.ts index 7ae233df6c1..30ea8fbe64f 100644 --- a/packages/firestore/test/unit/specs/existence_filter_spec.test.ts +++ b/packages/firestore/test/unit/specs/existence_filter_spec.test.ts @@ -74,7 +74,7 @@ describeSpec('Existence Filters:', [], () => { .watchAcksFull(query1, 1000, doc1) .expectEvents(query1, { added: [doc1] }) .userUnlistens(query1) - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [doc1], fromCache: true }) // The empty existence filter is ignored since Watch hasn't ACKed the // target diff --git a/packages/firestore/test/unit/specs/limbo_spec.test.ts b/packages/firestore/test/unit/specs/limbo_spec.test.ts index 8f1c824b42d..071815cb793 100644 --- a/packages/firestore/test/unit/specs/limbo_spec.test.ts +++ b/packages/firestore/test/unit/specs/limbo_spec.test.ts @@ -444,7 +444,7 @@ describeSpec('Limbo Documents:', [], () => { .expectEvents(query1, { fromCache: true }) .runTimer(TimerId.ClientMetadataRefresh) .expectPrimaryState(true) - .expectListen(query1, 'resume-token-1000000') + .expectListen(query1, { resumeToken: 'resume-token-1000000' }) .watchAcksFull(query1, 3 * 1e6) .expectLimboDocs(docB.key) .ackLimbo(4 * 1e6, deletedDocB) @@ -476,7 +476,7 @@ describeSpec('Limbo Documents:', [], () => { .expectLimboDocs(docB.key, docC.key) .client(1) .stealPrimaryLease() - .expectListen(query1, 'resume-token-1000000') + .expectListen(query1, { resumeToken: 'resume-token-1000000' }) .client(0) .runTimer(TimerId.ClientMetadataRefresh) .expectPrimaryState(false) @@ -489,7 +489,7 @@ describeSpec('Limbo Documents:', [], () => { .client(0) .expectEvents(query1, { removed: [docB], fromCache: true }) .stealPrimaryLease() - .expectListen(query1, 'resume-token-1000000') + .expectListen(query1, { resumeToken: 'resume-token-1000000' }) .watchAcksFull(query1, 5 * 1e6) .expectLimboDocs(docC.key) .ackLimbo(6 * 1e6, deletedDocC) @@ -545,7 +545,7 @@ describeSpec('Limbo Documents:', [], () => { // document `docBCommitted`, since we haven't received the resolved // document from Watch. Until we do, we return the version from cache // even though the backend told it does not match. - .userListens(originalQuery, 'resume-token-2000') + .userListens(originalQuery, { resumeToken: 'resume-token-2000' }) .expectEvents(originalQuery, { added: [docA, docBDirty], fromCache: true diff --git a/packages/firestore/test/unit/specs/limit_spec.test.ts b/packages/firestore/test/unit/specs/limit_spec.test.ts index 33ab16b90cb..16784697431 100644 --- a/packages/firestore/test/unit/specs/limit_spec.test.ts +++ b/packages/firestore/test/unit/specs/limit_spec.test.ts @@ -58,7 +58,7 @@ describeSpec('Limits:', [], () => { }) .userUnlistens(query1) .userSets('collection/c', { key: 'c' }) - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [doc1, doc2], fromCache: true @@ -203,7 +203,7 @@ describeSpec('Limits:', [], () => { .userUnlistens(limitQuery) .watchRemoves(limitQuery) .userSets('collection/a', { matches: false }) - .userListens(limitQuery, 'resume-token-1004') + .userListens(limitQuery, { resumeToken: 'resume-token-1004' }) .expectEvents(limitQuery, { added: [doc2, doc3], fromCache: true }); } ); @@ -238,7 +238,7 @@ describeSpec('Limits:', [], () => { .userUnlistens(limitQuery) .watchRemoves(limitQuery) .userSets('collection/a', { pos: 4 }) - .userListens(limitQuery, 'resume-token-1004') + .userListens(limitQuery, { resumeToken: 'resume-token-1004' }) .expectEvents(limitQuery, { added: [doc2, doc3], fromCache: true }); } ); @@ -273,11 +273,11 @@ describeSpec('Limits:', [], () => { .expectEvents(limitQuery, {}) .userUnlistens(limitQuery) .watchRemoves(limitQuery) - .userListens(fullQuery, 'resume-token-1003') + .userListens(fullQuery, { resumeToken: 'resume-token-1003' }) .expectEvents(fullQuery, { added: [doc1, doc2, doc3], fromCache: true }) .watchAcksFull(fullQuery, 1005, doc1Edited) .expectEvents(fullQuery, { modified: [doc1Edited] }) - .userListens(limitQuery, 'resume-token-1004') + .userListens(limitQuery, { resumeToken: 'resume-token-1004' }) .expectEvents(limitQuery, { added: [doc2, doc3], fromCache: true }); } ); @@ -332,7 +332,7 @@ describeSpec('Limits:', [], () => { // We listen to the limit query again. Note that we include // `firstDocument` in the local result since we did not resolve its // limbo state. - .userListens(limitQuery, 'resume-token-1001') + .userListens(limitQuery, { resumeToken: 'resume-token-1001' }) .expectEvents(limitQuery, { added: [firstDocument], fromCache: true }) .watchAcks(limitQuery) // Watch resumes the query from the provided resume token, but does @@ -400,7 +400,7 @@ describeSpec('Limits:', [], () => { .watchRemoves(fullQuery) // Re-issue the limit query and verify that we return `secondDocument` // from cache. - .userListens(limitQuery, 'resume-token-2001') + .userListens(limitQuery, { resumeToken: 'resume-token-2001' }) .expectEvents(limitQuery, { added: [secondDocument], fromCache: true diff --git a/packages/firestore/test/unit/specs/listen_spec.test.ts b/packages/firestore/test/unit/specs/listen_spec.test.ts index daa9c3f1217..ad0fa9ab201 100644 --- a/packages/firestore/test/unit/specs/listen_spec.test.ts +++ b/packages/firestore/test/unit/specs/listen_spec.test.ts @@ -278,7 +278,7 @@ describeSpec('Listens:', [], () => { // Remove and re-add listener. .userUnlistens(query1) .watchRemoves(query1) - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [docAv2], fromCache: true }) // watch sends old snapshot. .watchAcksFull(query1, 1000, docAv1) @@ -313,7 +313,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, { modified: [docAv2] }) // restart the client and re-listen. .restart() - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [docAv2], fromCache: true }) // watch sends old snapshot. .watchAcksFull(query1, 1000, docAv1) @@ -353,7 +353,7 @@ describeSpec('Listens:', [], () => { // us up to docAV2 since that's the last relevant change to the query // (the document falls out) and send us a snapshot that's ahead of // docAv3 (which is already in our cache). - .userListens(visibleQuery, 'resume-token-1000') + .userListens(visibleQuery, { resumeToken: 'resume-token-1000' }) .watchAcks(visibleQuery) .watchSends({ removed: [visibleQuery] }, docAv2) .watchCurrents(visibleQuery, 'resume-token-5000') @@ -362,7 +362,7 @@ describeSpec('Listens:', [], () => { .userUnlistens(visibleQuery) .watchRemoves(visibleQuery) // Listen to allQuery again and make sure we still get docAv3. - .userListens(allQuery, 'resume-token-4000') + .userListens(allQuery, { resumeToken: 'resume-token-4000' }) .expectEvents(allQuery, { added: [docAv3], fromCache: true }) .watchAcksFull(allQuery, 6000) .expectEvents(allQuery, { fromCache: false }) @@ -398,7 +398,7 @@ describeSpec('Listens:', [], () => { // us up to docAV2 since that's the last relevant change to the query // (the document falls out) and send us a snapshot that's ahead of // docAv3 (which is already in our cache). - .userListens(visibleQuery, 'resume-token-1000') + .userListens(visibleQuery, { resumeToken: 'resume-token-1000' }) .watchAcks(visibleQuery) .watchSends({ removed: [visibleQuery] }, docAv2) .watchCurrents(visibleQuery, 'resume-token-5000') @@ -407,7 +407,7 @@ describeSpec('Listens:', [], () => { .userUnlistens(visibleQuery) .watchRemoves(visibleQuery) // Listen to allQuery again and make sure we still get no docs. - .userListens(allQuery, 'resume-token-4000') + .userListens(allQuery, { resumeToken: 'resume-token-4000' }) .watchAcksFull(allQuery, 6000) .expectEvents(allQuery, { fromCache: false }) ); @@ -484,7 +484,7 @@ describeSpec('Listens:', [], () => { .userUnlistens(collQuery) .watchRemoves(collQuery) // Verify that DocA and DocB exists - .userListens(collQuery, 'resume-token-1000') + .userListens(collQuery, { resumeToken: 'resume-token-1000' }) .expectEvents(collQuery, { added: [docA, docB], fromCache: true }) .userUnlistens(collQuery) // Now send a document query that produces no results from the server @@ -498,7 +498,7 @@ describeSpec('Listens:', [], () => { .userUnlistens(docQuery) .watchRemoves(docQuery) // Re-add the initial collection query. Only Doc B exists now - .userListens(collQuery, 'resume-token-1000') + .userListens(collQuery, { resumeToken: 'resume-token-1000' }) .expectEvents(collQuery, { added: [docB], fromCache: true }) ); }); @@ -514,7 +514,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, { added: [docA] }) .userUnlistens(query1) .watchRemoves(query1) - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [docA], fromCache: true }) .watchAcks(query1) .watchSends({ removed: [query1] }, deletedDocA) @@ -536,7 +536,7 @@ describeSpec('Listens:', [], () => { .watchSends({ affects: [query1] }, docB) .watchSnapshots(2000) .watchRemoves(query1) - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [docA], fromCache: true }); }); @@ -560,7 +560,7 @@ describeSpec('Listens:', [], () => { // snapshot and don't synthesize a document delete. .expectEvents(query1, { fromCache: false }) .userUnlistens(query1) - .userListens(query1, 'resume-token-2000') + .userListens(query1, { resumeToken: 'resume-token-2000' }) .expectEvents(query1, { added: [docA], fromCache: true }) ); } @@ -596,7 +596,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, { added: [docA] }) .userUnlistens(query1) .watchRemoves(query1) - .userListens(query1, 'resume-token-2000') + .userListens(query1, { resumeToken: 'resume-token-2000' }) .expectEvents(query1, { added: [docA], fromCache: true }) .watchAcksFull(query1, 3000) .expectEvents(query1, {}); @@ -616,7 +616,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, { added: [docA] }) .userUnlistens(query1) .watchRemoves(query1) - .userListens(query1, 'resume-token-2000') + .userListens(query1, { resumeToken: 'resume-token-2000' }) .expectEvents(query1, { added: [docA], fromCache: true }) .watchAcksFull(query1, 3000) .expectEvents(query1, {}); @@ -639,7 +639,7 @@ describeSpec('Listens:', [], () => { .userUnlistens(query1) .watchRemoves(query1) - .userListens(query1, 'resume-token-2000') + .userListens(query1, { resumeToken: 'resume-token-2000' }) .expectEvents(query1, { added: [docA], fromCache: true }) .watchAcks(query1) .watchCurrents(query1, 'resume-token-3000') @@ -667,7 +667,7 @@ describeSpec('Listens:', [], () => { .watchSnapshots(2000, [], 'resume-token-2000') .restart() - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [docA], fromCache: true }) .watchAcks(query1) .watchCurrents(query1, 'resume-token-3000') @@ -700,7 +700,7 @@ describeSpec('Listens:', [], () => { .watchSnapshots(minutesLater, [], 'resume-token-minutes-later') .restart() - .userListens(query1, 'resume-token-minutes-later') + .userListens(query1, { resumeToken: 'resume-token-minutes-later' }) .expectEvents(query1, { added: [docA], fromCache: true }) .watchAcks(query1) .watchCurrents(query1, 'resume-token-even-later') @@ -862,7 +862,7 @@ describeSpec('Listens:', [], () => { .userListens(query1) .expectEvents(query1, { added: [docA], fromCache: true }) .client(0) - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docB) .client(1) .expectEvents(query1, { added: [docB] }); @@ -1133,7 +1133,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, { fromCache: true }) .client(0) .enableNetwork() - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000) .client(1) .expectEvents(query1, {}); @@ -1184,7 +1184,7 @@ describeSpec('Listens:', [], () => { .client(1) .expectEvents(query1, { fromCache: true }) .client(2) - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .expectPrimaryState(true) .watchAcksFull(query1, 2000) .client(0) @@ -1269,7 +1269,7 @@ describeSpec('Listens:', [], () => { .client(1) .runTimer(TimerId.ClientMetadataRefresh) .expectPrimaryState(true) - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docB) .expectEvents(query1, { added: [docB] }) .client(2) @@ -1299,7 +1299,7 @@ describeSpec('Listens:', [], () => { .client(1) .runTimer(TimerId.ClientMetadataRefresh) .expectPrimaryState(true) - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docB) .client(2) .expectEvents(query1, { added: [docB] }); @@ -1321,7 +1321,7 @@ describeSpec('Listens:', [], () => { .userListens(query1) .expectEvents(query1, { added: [docA] }) .stealPrimaryLease() - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docB) .expectEvents(query1, { added: [docB] }) .client(0) @@ -1359,7 +1359,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, { added: [docA] }) .client(2) .stealPrimaryLease() - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .client(1) .runTimer(TimerId.ClientMetadataRefresh) .expectPrimaryState(false) @@ -1369,7 +1369,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, { added: [docB] }) .client(1) .stealPrimaryLease() - .expectListen(query1, 'resume-token-2000') + .expectListen(query1, { resumeToken: 'resume-token-2000' }) .watchAcksFull(query1, 3000, docC) .client(0) .expectEvents(query1, { added: [docC] }); @@ -1427,7 +1427,7 @@ describeSpec('Listens:', [], () => { .shutdown() .client(2) .expectPrimaryState(true) - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docB) .client(1) .expectEvents(query1, { added: [docB] }); @@ -1448,7 +1448,7 @@ describeSpec('Listens:', [], () => { .expectEvents(query1, {}) .client(1) .stealPrimaryLease() - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docA) .shutdown() .client(0) @@ -1457,7 +1457,7 @@ describeSpec('Listens:', [], () => { // is already eligible to obtain it again. .runTimer(TimerId.ClientMetadataRefresh) .expectPrimaryState(true) - .expectListen(query1, 'resume-token-2000') + .expectListen(query1, { resumeToken: 'resume-token-2000' }) .expectEvents(query1, { added: [docA] }) ); } diff --git a/packages/firestore/test/unit/specs/orderby_spec.test.ts b/packages/firestore/test/unit/specs/orderby_spec.test.ts index f74e1594ae6..3bb8a4fa0e7 100644 --- a/packages/firestore/test/unit/specs/orderby_spec.test.ts +++ b/packages/firestore/test/unit/specs/orderby_spec.test.ts @@ -66,7 +66,7 @@ describeSpec('OrderBy:', [], () => { .expectEvents(query1, { added: [docB, docA] }) .userUnlistens(query1) .watchRemoves(query1) - .userListens(query1, 'resume-token-1002') + .userListens(query1, { resumeToken: 'resume-token-1002' }) .expectEvents(query1, { added: [docB, docA], fromCache: true }) .watchAcksFull(query1, 1002) .expectEvents(query1, {}); diff --git a/packages/firestore/test/unit/specs/perf_spec.test.ts b/packages/firestore/test/unit/specs/perf_spec.test.ts index 7fafa5e95f1..50346863cd2 100644 --- a/packages/firestore/test/unit/specs/perf_spec.test.ts +++ b/packages/firestore/test/unit/specs/perf_spec.test.ts @@ -234,7 +234,9 @@ describeSpec( .expectEvents(query1, { added: docs }) .userUnlistens(query1) .watchRemoves(query1) - .userListens(query1, 'resume-token-' + currentVersion) + .userListens(query1, { + resumeToken: 'resume-token-' + currentVersion + }) .expectEvents(query1, { added: docs, fromCache: true }) .watchAcksFull(query1, ++currentVersion) .expectEvents(query1, {}) diff --git a/packages/firestore/test/unit/specs/persistence_spec.test.ts b/packages/firestore/test/unit/specs/persistence_spec.test.ts index ac89459ce25..7beaf32d26c 100644 --- a/packages/firestore/test/unit/specs/persistence_spec.test.ts +++ b/packages/firestore/test/unit/specs/persistence_spec.test.ts @@ -79,7 +79,7 @@ describeSpec('Persistence:', [], () => { .watchAcksFull(query1, 1000, doc1) .expectEvents(query1, { added: [doc1] }) .restart() - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [doc1], fromCache: true }); }); @@ -94,7 +94,7 @@ describeSpec('Persistence:', [], () => { .expectEvents(query1, { added: [doc1] }) // Normally this would clear the cached remote documents. .userUnlistens(query1) - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [doc1], fromCache: true }) ); }); diff --git a/packages/firestore/test/unit/specs/recovery_spec.test.ts b/packages/firestore/test/unit/specs/recovery_spec.test.ts index c5da05ff28f..02dfa079d76 100644 --- a/packages/firestore/test/unit/specs/recovery_spec.test.ts +++ b/packages/firestore/test/unit/specs/recovery_spec.test.ts @@ -173,7 +173,7 @@ describeSpec('Persistence Recovery', ['no-ios', 'no-android'], () => { .recoverDatabase() .runTimer(TimerId.AsyncQueueRetry) .expectPrimaryState(true) - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docB) .expectEvents(query1, { added: [docB] }) ); @@ -214,7 +214,7 @@ describeSpec('Persistence Recovery', ['no-ios', 'no-android'], () => { .recoverDatabase() .runTimer(TimerId.AsyncQueueRetry) .expectPrimaryState(true) - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .watchAcksFull(query1, 2000, docB) .client(2) .expectEvents(query1, { added: [docB] }) @@ -506,7 +506,7 @@ describeSpec('Persistence Recovery', ['no-ios', 'no-android'], () => { .recoverDatabase() .userUnlistens(query1) // No event since the document was removed - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) ); } ); @@ -612,7 +612,7 @@ describeSpec('Persistence Recovery', ['no-ios', 'no-android'], () => { // Verify that `doc1Query` can be listened to again. Note that the // resume token is slightly outdated since we failed to persist the // target update during the release. - .userListens(doc1Query, 'resume-token-1000') + .userListens(doc1Query, { resumeToken: 'resume-token-1000' }) .expectEvents(doc1Query, { added: [doc1a], fromCache: true @@ -829,7 +829,7 @@ describeSpec('Persistence Recovery', ['no-ios', 'no-android'], () => { .userUnlistens(query1) .watchRemoves(query1) .recoverDatabase() - .userListens(query1, 'resume-token-1000') + .userListens(query1, { resumeToken: 'resume-token-1000' }) .expectEvents(query1, { added: [doc1], fromCache: true diff --git a/packages/firestore/test/unit/specs/resume_token_spec.test.ts b/packages/firestore/test/unit/specs/resume_token_spec.test.ts index 537b637fe39..cd0f5b9435a 100644 --- a/packages/firestore/test/unit/specs/resume_token_spec.test.ts +++ b/packages/firestore/test/unit/specs/resume_token_spec.test.ts @@ -51,7 +51,7 @@ describeSpec('Resume tokens:', [], () => { .watchSnapshots(1000) .expectEvents(query1, { added: [doc1] }) .userUnlistens(query1) - .userListens(query1, 'custom-query-resume-token') + .userListens(query1, { resumeToken: 'custom-query-resume-token' }) .expectEvents(query1, { fromCache: true, added: [doc1] }) .watchAcks(query1) .watchSnapshots(1001); diff --git a/packages/firestore/test/unit/specs/spec_builder.ts b/packages/firestore/test/unit/specs/spec_builder.ts index 87b514ee0b1..7eae0d3d593 100644 --- a/packages/firestore/test/unit/specs/spec_builder.ts +++ b/packages/firestore/test/unit/specs/spec_builder.ts @@ -256,8 +256,7 @@ export class SpecBuilder { userListens( query: Query, - resumeFrom?: string, - readTime?: TestSnapshotVersion + resume?: { resumeToken?: string; readTime?: TestSnapshotVersion } ): this { this.nextStep(); @@ -277,7 +276,12 @@ export class SpecBuilder { } this.queryMapping.set(target, targetId); - this.addQueryToActiveTargets(targetId, query, resumeFrom, readTime); + this.addQueryToActiveTargets( + targetId, + query, + resume?.resumeToken, + resume?.readTime + ); this.currentStep = { userListen: { targetId, query: SpecBuilder.queryToSpec(query) }, expectedState: { activeTargets: { ...this.activeTargets } } @@ -876,8 +880,7 @@ export class SpecBuilder { /** Registers a query that is active in another tab. */ expectListen( query: Query, - resumeToken?: string, - readTime?: TestSnapshotVersion + resume?: { resumeToken?: string; readTime?: TestSnapshotVersion } ): this { this.assertStep('Expectations require previous step'); @@ -885,7 +888,12 @@ export class SpecBuilder { const targetId = this.queryIdGenerator.cachedId(target); this.queryMapping.set(target, targetId); - this.addQueryToActiveTargets(targetId, query, resumeToken, readTime); + this.addQueryToActiveTargets( + targetId, + query, + resume?.resumeToken, + resume?.readTime + ); const currentStep = this.currentStep!; currentStep.expectedState = currentStep.expectedState || {}; diff --git a/packages/firestore/test/unit/specs/spec_test_runner.ts b/packages/firestore/test/unit/specs/spec_test_runner.ts index 46bdacaf8fa..e3cf3faba63 100644 --- a/packages/firestore/test/unit/specs/spec_test_runner.ts +++ b/packages/firestore/test/unit/specs/spec_test_runner.ts @@ -990,7 +990,10 @@ abstract class TestRunner { SnapshotVersion.min() ); } else { - targetData = targetData.withReadTime(version(expected.readTime!)); + targetData = targetData.withResumeToken( + ByteString.EMPTY_BYTE_STRING, + version(expected.readTime!) + ); } const expectedTarget = toTarget(this.serializer, targetData); expect(actualTarget.query).to.deep.equal(expectedTarget.query); diff --git a/packages/firestore/test/unit/specs/write_spec.test.ts b/packages/firestore/test/unit/specs/write_spec.test.ts index f9c5e3ed514..9c9fe8c39fa 100644 --- a/packages/firestore/test/unit/specs/write_spec.test.ts +++ b/packages/firestore/test/unit/specs/write_spec.test.ts @@ -1068,7 +1068,7 @@ describeSpec('Writes:', [], () => { // Start a new client. DocV1 still has pending writes. .client(1) .stealPrimaryLease() - .expectListen(query1, 'resume-token-1000') + .expectListen(query1, { resumeToken: 'resume-token-1000' }) .userListens(query2) .expectEvents(query2, { added: [docV1Committed],