From 9602712941d2a802d5b52f808bc9f8225c2a3681 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 15 May 2020 15:26:07 -0400 Subject: [PATCH 01/39] Renaming interfaces without leading I --- packages/firestore/src/local/indexeddb_schema.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/firestore/src/local/indexeddb_schema.ts b/packages/firestore/src/local/indexeddb_schema.ts index 2026b373adb..645aa7ec226 100644 --- a/packages/firestore/src/local/indexeddb_schema.ts +++ b/packages/firestore/src/local/indexeddb_schema.ts @@ -19,6 +19,7 @@ import { BatchId, ListenSequenceNumber, TargetId } from '../core/types'; import { ResourcePath } from '../model/path'; import { BundledQuery } from '../protos/firestore_bundle_proto'; import * as api from '../protos/firestore_proto_api'; +import { BundledQuery } from '../protos/firestore_bundle_proto'; import { hardAssert, debugAssert } from '../util/assert'; import { SnapshotVersion } from '../core/snapshot_version'; From c5e783ec310164486819a855ead96b4a2e95297c Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 21 May 2020 12:08:35 -0400 Subject: [PATCH 02/39] Initial commit of bundle reading - for web only. --- packages/firestore/src/util/bundle.ts | 215 +++++++++++ .../firestore/test/unit/util/bundle.test.ts | 353 ++++++++++++++++++ 2 files changed, 568 insertions(+) create mode 100644 packages/firestore/src/util/bundle.ts create mode 100644 packages/firestore/test/unit/util/bundle.test.ts diff --git a/packages/firestore/src/util/bundle.ts b/packages/firestore/src/util/bundle.ts new file mode 100644 index 00000000000..d11613361b0 --- /dev/null +++ b/packages/firestore/src/util/bundle.ts @@ -0,0 +1,215 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as api from '../protos/firestore_proto_api'; +import { + BundledDocumentMetadata, + BundledQuery, + BundleElement, + BundleMetadata +} from '../protos/firestore_bundle_proto'; + +/** + * A complete element in the bundle stream, together with the byte length it + * occupies in the stream. + */ +export class SizedBundleElement { + constructor( + public payload: BundledQuery | api.Document | BundledDocumentMetadata, + public byteLength: number + ) {} +} + +/** + * Create a `ReadableStream` from a underlying buffer. + * + * @param data: Underlying buffer. + * @param bytesPerRead: How many bytes to read from the underlying buffer from each read through the stream. + */ +export function toReadableStream( + data: Uint8Array | ArrayBuffer, + bytesPerRead = 10240 +): ReadableStream { + let readFrom = 0; + return new ReadableStream({ + start(controller) {}, + async pull(controller): Promise { + controller.enqueue(data.slice(readFrom, readFrom + bytesPerRead)); + readFrom += bytesPerRead; + if (readFrom >= data.byteLength) { + controller.close(); + } + } + }); +} + +/** + * A class representing a bundle. + * + * Takes a bundle stream or buffer, and presents abstractions to read bundled + * elements out of the underlying content. + */ +export class Bundle { + // Cached bundle metadata. + private metadata?: BundleMetadata | null; + // The reader instance of the given ReadableStream. + private reader: ReadableStreamDefaultReader; + // Internal buffer to hold bundle content, accumulating incomplete element content. + private buffer: Uint8Array = new Uint8Array(); + private textDecoder = new TextDecoder('utf-8'); + + constructor( + private bundleStream: + | ReadableStream + | Uint8Array + | ArrayBuffer + ) { + if ( + bundleStream instanceof Uint8Array || + bundleStream instanceof ArrayBuffer + ) { + this.bundleStream = toReadableStream(bundleStream); + } + this.reader = (this.bundleStream as ReadableStream).getReader(); + } + + /** + * Returns the metadata of the bundle. + */ + async getMetadata(): Promise { + if (!this.metadata) { + const result = await this.nextElement(); + if (result === null || result instanceof SizedBundleElement) { + throw new Error(`The first element is not metadata, it is ${result}`); + } + this.metadata = (result as BundleElement).metadata; + } + + return this.metadata!; + } + + /** + * Asynchronously iterate through all bundle elements (except bundle metadata). + */ + async *elements(): AsyncIterableIterator { + let element = await this.nextElement(); + while (element !== null) { + if (element instanceof SizedBundleElement) { + yield element; + } else { + this.metadata = element.metadata; + } + element = await this.nextElement(); + } + } + + // Reads from the head of internal buffer, and pulling more data from underlying stream if a complete element + // cannot be found, until an element(including the prefixed length and the JSON string) is found. + // + // Once a complete element is read, it is dropped from internal buffer. + // + // Returns either the bundled element, or null if we have reached the end of the stream. + private async nextElement(): Promise< + BundleElement | SizedBundleElement | null + > { + const lengthBuffer = await this.readLength(); + if (lengthBuffer === null) { + return null; + } + + const lengthString = this.textDecoder.decode(lengthBuffer); + const length = parseInt(lengthString, 10); + if (isNaN(length)) { + throw new Error(`length string (${lengthString}) is not valid number`); + } + + const jsonString = await this.readJsonString(lengthBuffer.length, length); + // Update the internal buffer to drop the read length and json string. + this.buffer = this.buffer.slice(lengthBuffer.length + length); + + if (!this.metadata) { + const element = JSON.parse(jsonString) as BundleElement; + return element; + } else { + return new SizedBundleElement( + JSON.parse(jsonString), + lengthBuffer.length + length + ); + } + } + + // First index of '{' from the underlying buffer. + private indexOfOpenBracket(): number { + return this.buffer.findIndex(v => v === 123); + } + + // Reads from the beginning of the inernal buffer, until the first '{', and return + // the content. + // If reached end of the stream, returns a null. + private async readLength(): Promise { + let position = this.indexOfOpenBracket(); + while (position < 0) { + const done = await this.pullMoreDataToBuffer(); + if (done) { + if (this.buffer.length === 0) { + return null; + } + position = this.indexOfOpenBracket(); + // Underlying stream is closed, and we still cannot find a '{'. + if (position < 0) { + throw new Error( + 'Reach to the end of bundle when a length string is expected.' + ); + } + } else { + position = this.indexOfOpenBracket(); + } + } + + return this.buffer.slice(0, position); + } + + // Reads from a specified position from the internal buffer, for a specified + // number of bytes, pulling more data from the underlying stream if needed. + // + // Returns a string decoded from the read bytes. + private async readJsonString(start: number, length: number): Promise { + while (this.buffer.length < start + length) { + const done = await this.pullMoreDataToBuffer(); + if (done) { + throw new Error('Reach to the end of bundle when more is expected.'); + } + } + + return this.textDecoder.decode(this.buffer.slice(start, start + length)); + } + + // Pulls more data from underlying stream to internal buffer. + // Returns a boolean indicating whether the stream is finished. + private async pullMoreDataToBuffer(): Promise { + const result = await this.reader.read(); + if (!result.done) { + const newBuffer = new Uint8Array( + this.buffer.length + result.value.length + ); + newBuffer.set(this.buffer); + newBuffer.set(result.value, this.buffer.length); + this.buffer = newBuffer; + } + return result.done; + } +} diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts new file mode 100644 index 00000000000..20b2c464ff9 --- /dev/null +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -0,0 +1,353 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { expect } from 'chai'; +import { Bundle, toReadableStream } from '../../../src/util/bundle'; + +const encoder = new TextEncoder(); + +function readableStreamFromString( + content: string, + bytesPerRead: number +): ReadableStream { + return toReadableStream(encoder.encode(content), bytesPerRead); +} + +function lengthPrefixedString(o: {}): string { + const str = JSON.stringify(o); + const l = encoder.encode(str).byteLength; + return `${l}${str}`; +} + +describe('readableStreamFromString()', () => { + it('returns stepping readable stream', async () => { + const encoder = new TextEncoder(); + const s = readableStreamFromString('0123456789', 4); + const r = s.getReader(); + + let result = await r.read(); + expect(result.value).to.deep.equal(encoder.encode('0123')); + expect(result.done).to.be.false; + + result = await r.read(); + expect(result.value).to.deep.equal(encoder.encode('4567')); + expect(result.done).to.be.false; + + result = await r.read(); + expect(result.value).to.deep.equal(encoder.encode('89')); + expect(result.done).to.be.false; + + result = await r.read(); + expect(result.value).to.be.undefined; + expect(result.done).to.be.true; + }); +}); + +describe.only('Bundle ', () => { + genericBundleReadingTests(1); + genericBundleReadingTests(4); + genericBundleReadingTests(64); + genericBundleReadingTests(1024); +}); + +function genericBundleReadingTests(bytesPerRead: number): void { + // Setting up test data. + const meta = { + metadata: { + id: 'test-bundle', + createTime: { seconds: 1577836805, nanos: 6 }, + version: 1, + totalDocuments: 1, + totalBytes: 416 + } + }; + const metaString = lengthPrefixedString(meta); + + const doc1Meta = { + documentMetadata: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc1', + readTime: { seconds: 5, nanos: 6 }, + exists: true + } + }; + const doc1MetaString = lengthPrefixedString(doc1Meta); + const doc1 = { + document: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc1', + createTime: { _seconds: 1, _nanoseconds: 2000000 }, + updateTime: { _seconds: 3, _nanoseconds: 4000 }, + fields: { foo: { stringValue: 'value' }, bar: { integerValue: -42 } } + } + }; + const doc1String = lengthPrefixedString(doc1); + + const doc2Meta = { + documentMetadata: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc2', + readTime: { seconds: 5, nanos: 6 }, + exists: true + } + }; + const doc2MetaString = lengthPrefixedString(doc2Meta); + const doc2 = { + document: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc2', + createTime: { _seconds: 1, _nanoseconds: 2000000 }, + updateTime: { _seconds: 3, _nanoseconds: 4000 }, + fields: { foo: { stringValue: 'value1' }, bar: { integerValue: 42 } } + } + }; + const doc2String = lengthPrefixedString(doc2); + + const noDocMeta = { + documentMetadata: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/nodoc', + readTime: { seconds: 5, nanos: 6 }, + exists: false + } + }; + const noDocMetaString = lengthPrefixedString(noDocMeta); + + const limitQuery = { + namedQuery: { + name: 'limitQuery', + bundledQuery: { + parent: 'projects/fireeats-97d5e/databases/(default)/documents', + structuredQuery: { + from: [{ collectionId: 'node_3.7.5_7Li7XoCjutvNxwD0tpo9' }], + orderBy: [{ field: { fieldPath: 'sort' }, direction: 'DESCENDING' }], + limit: { 'value': 1 } + }, + limitType: 'FIRST' + }, + readTime: { 'seconds': 1590011379, 'nanos': 191164000 } + } + }; + const limitQueryString = lengthPrefixedString(limitQuery); + const limitToLastQuery = { + namedQuery: { + name: 'limitToLastQuery', + bundledQuery: { + parent: 'projects/fireeats-97d5e/databases/(default)/documents', + structuredQuery: { + from: [{ collectionId: 'node_3.7.5_7Li7XoCjutvNxwD0tpo9' }], + orderBy: [{ field: { fieldPath: 'sort' }, direction: 'ASCENDING' }], + limit: { 'value': 1 } + }, + limitType: 'LAST' + }, + readTime: { 'seconds': 1590011379, 'nanos': 543063000 } + } + }; + const limitToLastQueryString = lengthPrefixedString(limitToLastQuery); + + async function expectErrorFromBundle( + bundleString: string, + bytesPerRead: number, + validMeta = false + ): Promise { + const bundleStream = readableStreamFromString(bundleString, bytesPerRead); + const bundle = new Bundle(bundleStream); + + if (!validMeta) { + await expect(await bundle.getMetadata()).should.be.rejected; + } else { + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + } + + const actual = []; + for await (const sizedElement of bundle.elements()) { + actual.push(sizedElement); + } + } + + it('reads with query and doc with bytesPerRead ' + bytesPerRead, async () => { + const bundleStream = readableStreamFromString( + metaString + + limitQueryString + + limitToLastQueryString + + doc1MetaString + + doc1String, + bytesPerRead + ); + const bundle = new Bundle(bundleStream); + + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + + const actual = []; + for await (const sizedElement of bundle.elements()) { + actual.push(sizedElement); + } + expect(actual.length).to.equal(4); + expect(actual[0]).to.deep.equal({ + payload: limitQuery, + byteLength: encoder.encode(limitQueryString).byteLength + }); + expect(actual[1]).to.deep.equal({ + payload: limitToLastQuery, + byteLength: encoder.encode(limitToLastQueryString).byteLength + }); + expect(actual[2]).to.deep.equal({ + payload: doc1Meta, + byteLength: encoder.encode(doc1MetaString).byteLength + }); + expect(actual[3]).to.deep.equal({ + payload: doc1, + byteLength: encoder.encode(doc1String).byteLength + }); + }); + + it( + 'reads with unexpected orders with bytesPerRead ' + bytesPerRead, + async () => { + const bundleStream = readableStreamFromString( + metaString + + doc1MetaString + + doc1String + + limitQueryString + + doc2MetaString + + doc2String, + bytesPerRead + ); + const bundle = new Bundle(bundleStream); + + const actual = []; + for await (const sizedElement of bundle.elements()) { + actual.push(sizedElement); + } + expect(actual.length).to.equal(5); + expect(actual[0]).to.deep.equal({ + payload: doc1Meta, + byteLength: encoder.encode(doc1MetaString).byteLength + }); + expect(actual[1]).to.deep.equal({ + payload: doc1, + byteLength: encoder.encode(doc1String).byteLength + }); + expect(actual[2]).to.deep.equal({ + payload: limitQuery, + byteLength: encoder.encode(limitQueryString).byteLength + }); + expect(actual[3]).to.deep.equal({ + payload: doc2Meta, + byteLength: encoder.encode(doc2MetaString).byteLength + }); + expect(actual[4]).to.deep.equal({ + payload: doc2, + byteLength: encoder.encode(doc2String).byteLength + }); + + // Reading metadata after other elements should also work. + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + } + ); + + it( + 'reads without named query with bytesPerRead ' + bytesPerRead, + async () => { + const bundleStream = readableStreamFromString( + metaString + doc1MetaString + doc1String, + bytesPerRead + ); + const bundle = new Bundle(bundleStream); + + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + + const actual = []; + for await (const sizedElement of bundle.elements()) { + actual.push(sizedElement); + } + expect(actual.length).to.equal(2); + expect(actual[0]).to.deep.equal({ + payload: doc1Meta, + byteLength: encoder.encode(doc1MetaString).byteLength + }); + expect(actual[1]).to.deep.equal({ + payload: doc1, + byteLength: encoder.encode(doc1String).byteLength + }); + } + ); + + it('reads with deleted doc with bytesPerRead ' + bytesPerRead, async () => { + const bundleStream = readableStreamFromString( + metaString + noDocMetaString + doc1MetaString + doc1String, + bytesPerRead + ); + const bundle = new Bundle(bundleStream); + + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + + const actual = []; + for await (const sizedElement of bundle.elements()) { + actual.push(sizedElement); + } + expect(actual.length).to.equal(3); + expect(actual[0]).to.deep.equal({ + payload: noDocMeta, + byteLength: encoder.encode(noDocMetaString).byteLength + }); + expect(actual[1]).to.deep.equal({ + payload: doc1Meta, + byteLength: encoder.encode(doc1MetaString).byteLength + }); + expect(actual[2]).to.deep.equal({ + payload: doc1, + byteLength: encoder.encode(doc1String).byteLength + }); + }); + + it( + 'reads without documents or query with bytesPerRead ' + bytesPerRead, + async () => { + const bundleStream = readableStreamFromString(metaString, bytesPerRead); + const bundle = new Bundle(bundleStream); + + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + + const actual = []; + for await (const sizedElement of bundle.elements()) { + actual.push(sizedElement); + } + expect(actual.length).to.equal(0); + } + ); + + it( + 'throws with ill-formatted bundle with bytesPerRead ' + bytesPerRead, + async () => { + await expect( + expectErrorFromBundle('metadata: "no length prefix"', bytesPerRead) + ).to.be.rejected; + + await expect( + expectErrorFromBundle('{metadata: "no length prefix"}', bytesPerRead) + ).to.be.rejected; + + await expect( + expectErrorFromBundle(metaString + 'invalid-string', bytesPerRead, true) + ).to.be.rejected; + + await expect(expectErrorFromBundle('1' + metaString, bytesPerRead)).to.be + .rejected; + } + ); +} From 5e7fb89aad81aa52661bb1c4e419da8c32aef6d5 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 21 May 2020 13:25:12 -0400 Subject: [PATCH 03/39] Tests only run when it is not Node. --- packages/firestore/test/unit/util/bundle.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index 20b2c464ff9..6083bd81a7d 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -16,6 +16,7 @@ */ import { expect } from 'chai'; import { Bundle, toReadableStream } from '../../../src/util/bundle'; +import { isNode } from '../../util/test_platform'; const encoder = new TextEncoder(); @@ -32,7 +33,8 @@ function lengthPrefixedString(o: {}): string { return `${l}${str}`; } -describe('readableStreamFromString()', () => { +// eslint-disable-next-line no-restricted-properties +(isNode() ? describe.skip : describe)('readableStreamFromString()', () => { it('returns stepping readable stream', async () => { const encoder = new TextEncoder(); const s = readableStreamFromString('0123456789', 4); @@ -56,7 +58,8 @@ describe('readableStreamFromString()', () => { }); }); -describe.only('Bundle ', () => { +// eslint-disable-next-line no-restricted-properties +(isNode() ? describe.skip : describe)('Bundle ', () => { genericBundleReadingTests(1); genericBundleReadingTests(4); genericBundleReadingTests(64); From 1ee161546c90869ac4105ced02656831b1e6e63a Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 21 May 2020 13:51:41 -0400 Subject: [PATCH 04/39] Fix redundant imports --- packages/firestore/src/local/indexeddb_schema.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/firestore/src/local/indexeddb_schema.ts b/packages/firestore/src/local/indexeddb_schema.ts index 645aa7ec226..2026b373adb 100644 --- a/packages/firestore/src/local/indexeddb_schema.ts +++ b/packages/firestore/src/local/indexeddb_schema.ts @@ -19,7 +19,6 @@ import { BatchId, ListenSequenceNumber, TargetId } from '../core/types'; import { ResourcePath } from '../model/path'; import { BundledQuery } from '../protos/firestore_bundle_proto'; import * as api from '../protos/firestore_proto_api'; -import { BundledQuery } from '../protos/firestore_bundle_proto'; import { hardAssert, debugAssert } from '../util/assert'; import { SnapshotVersion } from '../core/snapshot_version'; From 18f0be1d843adbc4f2ebe921111307fa344f2f03 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 21 May 2020 14:35:54 -0400 Subject: [PATCH 05/39] Fix missing textencoder --- packages/firestore/test/unit/util/bundle.test.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index 6083bd81a7d..39c705dc647 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -18,18 +18,16 @@ import { expect } from 'chai'; import { Bundle, toReadableStream } from '../../../src/util/bundle'; import { isNode } from '../../util/test_platform'; -const encoder = new TextEncoder(); - function readableStreamFromString( content: string, bytesPerRead: number ): ReadableStream { - return toReadableStream(encoder.encode(content), bytesPerRead); + return toReadableStream(new TextEncoder().encode(content), bytesPerRead); } function lengthPrefixedString(o: {}): string { const str = JSON.stringify(o); - const l = encoder.encode(str).byteLength; + const l = new TextEncoder().encode(str).byteLength; return `${l}${str}`; } From aa455bff1db26c14b37914f202ddf59cb55e4965 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 29 May 2020 13:59:20 -0400 Subject: [PATCH 06/39] Remove generator. --- .../src/util/{bundle.ts => bundle_reader.ts} | 114 ++++++------ .../firestore/test/unit/util/bundle.test.ts | 162 ++++++++---------- 2 files changed, 134 insertions(+), 142 deletions(-) rename packages/firestore/src/util/{bundle.ts => bundle_reader.ts} (66%) diff --git a/packages/firestore/src/util/bundle.ts b/packages/firestore/src/util/bundle_reader.ts similarity index 66% rename from packages/firestore/src/util/bundle.ts rename to packages/firestore/src/util/bundle_reader.ts index d11613361b0..8bfb19f8e5d 100644 --- a/packages/firestore/src/util/bundle.ts +++ b/packages/firestore/src/util/bundle_reader.ts @@ -15,10 +15,7 @@ * limitations under the License. */ -import * as api from '../protos/firestore_proto_api'; import { - BundledDocumentMetadata, - BundledQuery, BundleElement, BundleMetadata } from '../protos/firestore_bundle_proto'; @@ -29,9 +26,14 @@ import { */ export class SizedBundleElement { constructor( - public payload: BundledQuery | api.Document | BundledDocumentMetadata, - public byteLength: number + public readonly payload: BundleElement, + // How many bytes this element takes to store in the bundle. + public readonly byteLength: number ) {} + + isBundleMetadata(): boolean { + return 'metadata' in this.payload; + } } /** @@ -63,7 +65,7 @@ export function toReadableStream( * Takes a bundle stream or buffer, and presents abstractions to read bundled * elements out of the underlying content. */ -export class Bundle { +export class BundleReader { // Cached bundle metadata. private metadata?: BundleMetadata | null; // The reader instance of the given ReadableStream. @@ -92,87 +94,91 @@ export class Bundle { */ async getMetadata(): Promise { if (!this.metadata) { - const result = await this.nextElement(); - if (result === null || result instanceof SizedBundleElement) { - throw new Error(`The first element is not metadata, it is ${result}`); - } - this.metadata = (result as BundleElement).metadata; + await this.nextElement(); } return this.metadata!; } /** - * Asynchronously iterate through all bundle elements (except bundle metadata). + * Returns the next BundleElement (together with its byte size in the bundle) + * that has not been read from underlying ReadableStream. Returns null if we + * have reached the end of the stream. + * + * Throws an error if the first element is not a BundleMetadata. */ - async *elements(): AsyncIterableIterator { - let element = await this.nextElement(); - while (element !== null) { - if (element instanceof SizedBundleElement) { - yield element; + async nextElement(): Promise { + const element = await this.readNextElement(); + if (!element) { + return element; + } + + if (!this.metadata) { + if (element.isBundleMetadata()) { + this.metadata = element.payload.metadata; } else { - this.metadata = element.metadata; + this.raiseError( + `The first element of the bundle is not a metadata, it is ${JSON.stringify( + element.payload + )}` + ); } - element = await this.nextElement(); } + + return element; } - // Reads from the head of internal buffer, and pulling more data from underlying stream if a complete element - // cannot be found, until an element(including the prefixed length and the JSON string) is found. - // - // Once a complete element is read, it is dropped from internal buffer. - // - // Returns either the bundled element, or null if we have reached the end of the stream. - private async nextElement(): Promise< - BundleElement | SizedBundleElement | null - > { + /** + * Reads from the head of internal buffer, and pulling more data from underlying stream if a complete element + * cannot be found, until an element(including the prefixed length and the JSON string) is found. + * + * Once a complete element is read, it is dropped from internal buffer. + * + * Returns either the bundled element, or null if we have reached the end of the stream. + */ + private async readNextElement(): Promise { const lengthBuffer = await this.readLength(); if (lengthBuffer === null) { return null; } const lengthString = this.textDecoder.decode(lengthBuffer); - const length = parseInt(lengthString, 10); + const length = Number(lengthString); if (isNaN(length)) { - throw new Error(`length string (${lengthString}) is not valid number`); + this.raiseError(`length string (${lengthString}) is not valid number`); } const jsonString = await this.readJsonString(lengthBuffer.length, length); // Update the internal buffer to drop the read length and json string. this.buffer = this.buffer.slice(lengthBuffer.length + length); - if (!this.metadata) { - const element = JSON.parse(jsonString) as BundleElement; - return element; - } else { - return new SizedBundleElement( - JSON.parse(jsonString), - lengthBuffer.length + length - ); - } + return new SizedBundleElement( + JSON.parse(jsonString), + lengthBuffer.length + length + ); } // First index of '{' from the underlying buffer. private indexOfOpenBracket(): number { - return this.buffer.findIndex(v => v === 123); + return this.buffer.findIndex(v => v === '{'.charCodeAt(0)); } - // Reads from the beginning of the inernal buffer, until the first '{', and return + // Reads from the beginning of the internal buffer, until the first '{', and return // the content. // If reached end of the stream, returns a null. private async readLength(): Promise { let position = this.indexOfOpenBracket(); while (position < 0) { - const done = await this.pullMoreDataToBuffer(); - if (done) { + const bytesRead = await this.pullMoreDataToBuffer(); + if (bytesRead < 0) { if (this.buffer.length === 0) { return null; } position = this.indexOfOpenBracket(); // Underlying stream is closed, and we still cannot find a '{'. if (position < 0) { - throw new Error( - 'Reach to the end of bundle when a length string is expected.' + this.raiseError( + 'Reached the end of bundle when a length string is expected.' ); } } else { @@ -189,20 +195,28 @@ export class Bundle { // Returns a string decoded from the read bytes. private async readJsonString(start: number, length: number): Promise { while (this.buffer.length < start + length) { - const done = await this.pullMoreDataToBuffer(); - if (done) { - throw new Error('Reach to the end of bundle when more is expected.'); + const bytesRead = await this.pullMoreDataToBuffer(); + if (bytesRead < 0) { + this.raiseError('Reached the end of bundle when more is expected.'); } } return this.textDecoder.decode(this.buffer.slice(start, start + length)); } + private raiseError(message: string): void { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + this.reader.cancel('Invalid bundle format.'); + throw new Error(message); + } + // Pulls more data from underlying stream to internal buffer. // Returns a boolean indicating whether the stream is finished. - private async pullMoreDataToBuffer(): Promise { + private async pullMoreDataToBuffer(): Promise { const result = await this.reader.read(); + let bytesRead = -1; if (!result.done) { + bytesRead = result.value.length; const newBuffer = new Uint8Array( this.buffer.length + result.value.length ); @@ -210,6 +224,6 @@ export class Bundle { newBuffer.set(result.value, this.buffer.length); this.buffer = newBuffer; } - return result.done; + return bytesRead; } } diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index 39c705dc647..6e40459139a 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -15,7 +15,11 @@ * limitations under the License. */ import { expect } from 'chai'; -import { Bundle, toReadableStream } from '../../../src/util/bundle'; +import { + BundleReader, + SizedBundleElement, + toReadableStream +} from '../../../src/util/bundle_reader'; import { isNode } from '../../util/test_platform'; function readableStreamFromString( @@ -65,6 +69,7 @@ function lengthPrefixedString(o: {}): string { }); function genericBundleReadingTests(bytesPerRead: number): void { + const encoder = new TextEncoder(); // Setting up test data. const meta = { metadata: { @@ -160,13 +165,41 @@ function genericBundleReadingTests(bytesPerRead: number): void { }; const limitToLastQueryString = lengthPrefixedString(limitToLastQuery); - async function expectErrorFromBundle( + async function getAllElement( + bundle: BundleReader + ): Promise { + const result: SizedBundleElement[] = []; + while (true) { + const sizedElement = await bundle.nextElement(); + if (sizedElement === null) { + break; + } + if (!sizedElement.isBundleMetadata()) { + result.push(sizedElement); + } + } + + return Promise.resolve(result); + } + + function verifySizedElement( + element: SizedBundleElement, + payload: unknown, + payloadString: string + ): void { + expect(element.payload).to.deep.equal(payload); + expect(element.byteLength).to.equal( + encoder.encode(payloadString).byteLength + ); + } + + async function parseThroughBundle( bundleString: string, bytesPerRead: number, validMeta = false ): Promise { const bundleStream = readableStreamFromString(bundleString, bytesPerRead); - const bundle = new Bundle(bundleStream); + const bundle = new BundleReader(bundleStream); if (!validMeta) { await expect(await bundle.getMetadata()).should.be.rejected; @@ -174,10 +207,7 @@ function genericBundleReadingTests(bytesPerRead: number): void { expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); } - const actual = []; - for await (const sizedElement of bundle.elements()) { - actual.push(sizedElement); - } + await getAllElement(bundle); } it('reads with query and doc with bytesPerRead ' + bytesPerRead, async () => { @@ -189,31 +219,16 @@ function genericBundleReadingTests(bytesPerRead: number): void { doc1String, bytesPerRead ); - const bundle = new Bundle(bundleStream); + const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); - const actual = []; - for await (const sizedElement of bundle.elements()) { - actual.push(sizedElement); - } + const actual = await getAllElement(bundle); expect(actual.length).to.equal(4); - expect(actual[0]).to.deep.equal({ - payload: limitQuery, - byteLength: encoder.encode(limitQueryString).byteLength - }); - expect(actual[1]).to.deep.equal({ - payload: limitToLastQuery, - byteLength: encoder.encode(limitToLastQueryString).byteLength - }); - expect(actual[2]).to.deep.equal({ - payload: doc1Meta, - byteLength: encoder.encode(doc1MetaString).byteLength - }); - expect(actual[3]).to.deep.equal({ - payload: doc1, - byteLength: encoder.encode(doc1String).byteLength - }); + verifySizedElement(actual[0], limitQuery, limitQueryString); + verifySizedElement(actual[1], limitToLastQuery, limitToLastQueryString); + verifySizedElement(actual[2], doc1Meta, doc1MetaString); + verifySizedElement(actual[3], doc1, doc1String); }); it( @@ -228,33 +243,15 @@ function genericBundleReadingTests(bytesPerRead: number): void { doc2String, bytesPerRead ); - const bundle = new Bundle(bundleStream); + const bundle = new BundleReader(bundleStream); - const actual = []; - for await (const sizedElement of bundle.elements()) { - actual.push(sizedElement); - } + const actual = await getAllElement(bundle); expect(actual.length).to.equal(5); - expect(actual[0]).to.deep.equal({ - payload: doc1Meta, - byteLength: encoder.encode(doc1MetaString).byteLength - }); - expect(actual[1]).to.deep.equal({ - payload: doc1, - byteLength: encoder.encode(doc1String).byteLength - }); - expect(actual[2]).to.deep.equal({ - payload: limitQuery, - byteLength: encoder.encode(limitQueryString).byteLength - }); - expect(actual[3]).to.deep.equal({ - payload: doc2Meta, - byteLength: encoder.encode(doc2MetaString).byteLength - }); - expect(actual[4]).to.deep.equal({ - payload: doc2, - byteLength: encoder.encode(doc2String).byteLength - }); + verifySizedElement(actual[0], doc1Meta, doc1MetaString); + verifySizedElement(actual[1], doc1, doc1String); + verifySizedElement(actual[2], limitQuery, limitQueryString); + verifySizedElement(actual[3], doc2Meta, doc2MetaString); + verifySizedElement(actual[4], doc2, doc2String); // Reading metadata after other elements should also work. expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); @@ -268,23 +265,14 @@ function genericBundleReadingTests(bytesPerRead: number): void { metaString + doc1MetaString + doc1String, bytesPerRead ); - const bundle = new Bundle(bundleStream); + const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); - const actual = []; - for await (const sizedElement of bundle.elements()) { - actual.push(sizedElement); - } + const actual = await getAllElement(bundle); expect(actual.length).to.equal(2); - expect(actual[0]).to.deep.equal({ - payload: doc1Meta, - byteLength: encoder.encode(doc1MetaString).byteLength - }); - expect(actual[1]).to.deep.equal({ - payload: doc1, - byteLength: encoder.encode(doc1String).byteLength - }); + verifySizedElement(actual[0], doc1Meta, doc1MetaString); + verifySizedElement(actual[1], doc1, doc1String); } ); @@ -293,41 +281,26 @@ function genericBundleReadingTests(bytesPerRead: number): void { metaString + noDocMetaString + doc1MetaString + doc1String, bytesPerRead ); - const bundle = new Bundle(bundleStream); + const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); - const actual = []; - for await (const sizedElement of bundle.elements()) { - actual.push(sizedElement); - } + const actual = await getAllElement(bundle); expect(actual.length).to.equal(3); - expect(actual[0]).to.deep.equal({ - payload: noDocMeta, - byteLength: encoder.encode(noDocMetaString).byteLength - }); - expect(actual[1]).to.deep.equal({ - payload: doc1Meta, - byteLength: encoder.encode(doc1MetaString).byteLength - }); - expect(actual[2]).to.deep.equal({ - payload: doc1, - byteLength: encoder.encode(doc1String).byteLength - }); + verifySizedElement(actual[0], noDocMeta, noDocMetaString); + verifySizedElement(actual[1], doc1Meta, doc1MetaString); + verifySizedElement(actual[2], doc1, doc1String); }); it( 'reads without documents or query with bytesPerRead ' + bytesPerRead, async () => { const bundleStream = readableStreamFromString(metaString, bytesPerRead); - const bundle = new Bundle(bundleStream); + const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); - const actual = []; - for await (const sizedElement of bundle.elements()) { - actual.push(sizedElement); - } + const actual = await getAllElement(bundle); expect(actual.length).to.equal(0); } ); @@ -336,19 +309,24 @@ function genericBundleReadingTests(bytesPerRead: number): void { 'throws with ill-formatted bundle with bytesPerRead ' + bytesPerRead, async () => { await expect( - expectErrorFromBundle('metadata: "no length prefix"', bytesPerRead) + parseThroughBundle('metadata: "no length prefix"', bytesPerRead) ).to.be.rejected; await expect( - expectErrorFromBundle('{metadata: "no length prefix"}', bytesPerRead) + parseThroughBundle('{metadata: "no length prefix"}', bytesPerRead) ).to.be.rejected; await expect( - expectErrorFromBundle(metaString + 'invalid-string', bytesPerRead, true) + parseThroughBundle(metaString + 'invalid-string', bytesPerRead, true) ).to.be.rejected; - await expect(expectErrorFromBundle('1' + metaString, bytesPerRead)).to.be + await expect(parseThroughBundle('1' + metaString, bytesPerRead)).to.be .rejected; + + // First element is not BundleMetadata. + await expect( + parseThroughBundle(doc1MetaString + doc1String, bytesPerRead) + ).to.be.rejected; } ); } From 78248cd9b73b569d9cd1e90bfd8e226ff6c18499 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Sat, 23 May 2020 18:53:53 -0400 Subject: [PATCH 07/39] Support bundle reader for Node --- packages/firestore/src/platform/platform.ts | 32 ++++++++++++++++ .../src/platform_browser/browser_platform.ts | 27 ++++++++++++- .../src/platform_node/node_platform.ts | 13 ++++++- packages/firestore/src/util/bundle_reader.ts | 38 +++---------------- packages/firestore/test/util/test_platform.ts | 6 ++- 5 files changed, 81 insertions(+), 35 deletions(-) diff --git a/packages/firestore/src/platform/platform.ts b/packages/firestore/src/platform/platform.ts index 1ab582ea05f..f5c7e013713 100644 --- a/packages/firestore/src/platform/platform.ts +++ b/packages/firestore/src/platform/platform.ts @@ -50,6 +50,8 @@ export interface Platform { */ randomBytes(nBytes: number): Uint8Array; + toByteStreamReader(source: unknown): ByteStreamReader; + /** The Platform's 'window' implementation or null if not available. */ readonly window: Window | null; @@ -60,6 +62,36 @@ export interface Platform { readonly base64Available: boolean; } +export interface ByteStreamReader { + read(): Promise; +} + +export interface ByteStreamReadResult { + done: boolean; + value: Uint8Array; +} + +export function toByteStreamReader( + source: Uint8Array, + bytesPerRead = 10240 +): ByteStreamReader { + let readFrom = 0; + return new (class implements ByteStreamReader { + async read(): Promise { + if (readFrom < source.byteLength) { + const result = { + value: source.slice(readFrom, readFrom + bytesPerRead), + done: false + }; + readFrom += bytesPerRead; + return result; + } + + return { value: new Uint8Array(), done: true }; + } + })(); +} + /** * Provides singleton helpers where setup code can inject a platform at runtime. * setPlatform needs to be set before Firestore is used and must be set exactly diff --git a/packages/firestore/src/platform_browser/browser_platform.ts b/packages/firestore/src/platform_browser/browser_platform.ts index 38133db4df5..54be8b25cf5 100644 --- a/packages/firestore/src/platform_browser/browser_platform.ts +++ b/packages/firestore/src/platform_browser/browser_platform.ts @@ -16,7 +16,12 @@ */ import { DatabaseId, DatabaseInfo } from '../core/database_info'; -import { Platform } from '../platform/platform'; +import { + ByteStreamReader, + ByteStreamReadResult, + Platform, + toByteStreamReader +} from '../platform/platform'; import { Connection } from '../remote/connection'; import { JsonProtoSerializer } from '../remote/serializer'; import { ConnectivityMonitor } from './../remote/connectivity_monitor'; @@ -93,4 +98,24 @@ export class BrowserPlatform implements Platform { } return bytes; } + + toByteStreamReader(source: unknown): ByteStreamReader { + if (source instanceof Uint8Array) { + return toByteStreamReader(source); + } + if (source instanceof ArrayBuffer) { + return toByteStreamReader(new Uint8Array(source)); + } + if (source instanceof ReadableStream) { + const reader = source.getReader(); + return new (class implements ByteStreamReader { + read(): Promise { + return reader.read(); + } + })(); + } + throw new Error( + 'Source of `toByteStreamReader` has to be Uint8Array, ArrayBuffer or ReadableStream' + ); + } } diff --git a/packages/firestore/src/platform_node/node_platform.ts b/packages/firestore/src/platform_node/node_platform.ts index 2fe74f7d68b..c2a40b76aa9 100644 --- a/packages/firestore/src/platform_node/node_platform.ts +++ b/packages/firestore/src/platform_node/node_platform.ts @@ -19,7 +19,11 @@ import { randomBytes } from 'crypto'; import { inspect } from 'util'; import { DatabaseId, DatabaseInfo } from '../core/database_info'; -import { Platform } from '../platform/platform'; +import { + ByteStreamReader, + Platform, + toByteStreamReader +} from '../platform/platform'; import { Connection } from '../remote/connection'; import { JsonProtoSerializer } from '../remote/serializer'; import { Code, FirestoreError } from '../util/error'; @@ -83,4 +87,11 @@ export class NodePlatform implements Platform { return randomBytes(nBytes); } + + toByteStreamReader(source: unknown): ByteStreamReader { + if (source instanceof Uint8Array) { + return toByteStreamReader(source); + } + throw new Error('Source of `toByteStreamReader` has to be Uint8Array'); + } } diff --git a/packages/firestore/src/util/bundle_reader.ts b/packages/firestore/src/util/bundle_reader.ts index 8bfb19f8e5d..4b28467a706 100644 --- a/packages/firestore/src/util/bundle_reader.ts +++ b/packages/firestore/src/util/bundle_reader.ts @@ -19,6 +19,7 @@ import { BundleElement, BundleMetadata } from '../protos/firestore_bundle_proto'; +import { ByteStreamReader, PlatformSupport } from '../platform/platform'; /** * A complete element in the bundle stream, together with the byte length it @@ -36,29 +37,6 @@ export class SizedBundleElement { } } -/** - * Create a `ReadableStream` from a underlying buffer. - * - * @param data: Underlying buffer. - * @param bytesPerRead: How many bytes to read from the underlying buffer from each read through the stream. - */ -export function toReadableStream( - data: Uint8Array | ArrayBuffer, - bytesPerRead = 10240 -): ReadableStream { - let readFrom = 0; - return new ReadableStream({ - start(controller) {}, - async pull(controller): Promise { - controller.enqueue(data.slice(readFrom, readFrom + bytesPerRead)); - readFrom += bytesPerRead; - if (readFrom >= data.byteLength) { - controller.close(); - } - } - }); -} - /** * A class representing a bundle. * @@ -68,8 +46,8 @@ export function toReadableStream( export class BundleReader { // Cached bundle metadata. private metadata?: BundleMetadata | null; - // The reader instance of the given ReadableStream. - private reader: ReadableStreamDefaultReader; + // The reader to read from underlying binary bundle data source. + private reader: ByteStreamReader; // Internal buffer to hold bundle content, accumulating incomplete element content. private buffer: Uint8Array = new Uint8Array(); private textDecoder = new TextDecoder('utf-8'); @@ -80,13 +58,9 @@ export class BundleReader { | Uint8Array | ArrayBuffer ) { - if ( - bundleStream instanceof Uint8Array || - bundleStream instanceof ArrayBuffer - ) { - this.bundleStream = toReadableStream(bundleStream); - } - this.reader = (this.bundleStream as ReadableStream).getReader(); + this.reader = PlatformSupport.getPlatform().toByteStreamReader( + bundleStream + ); } /** diff --git a/packages/firestore/test/util/test_platform.ts b/packages/firestore/test/util/test_platform.ts index cd9d458f63d..b4c023d1907 100644 --- a/packages/firestore/test/util/test_platform.ts +++ b/packages/firestore/test/util/test_platform.ts @@ -16,7 +16,7 @@ */ import { DatabaseId, DatabaseInfo } from '../../src/core/database_info'; -import { Platform } from '../../src/platform/platform'; +import { ByteStreamReader, Platform } from '../../src/platform/platform'; import { Connection } from '../../src/remote/connection'; import { JsonProtoSerializer } from '../../src/remote/serializer'; import { debugAssert, fail } from '../../src/util/assert'; @@ -270,6 +270,10 @@ export class TestPlatform implements Platform { randomBytes(nBytes: number): Uint8Array { return this.basePlatform.randomBytes(nBytes); } + + toByteStreamReader(source: unknown): ByteStreamReader { + return this.basePlatform.toByteStreamReader(source); + } } /** Returns true if we are running under Node. */ From 83160a1f3b51b0cc19a0057b5d30109f9b6bf186 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 29 May 2020 14:29:05 -0400 Subject: [PATCH 08/39] Fix rebase errors. --- packages/firestore/src/platform/platform.ts | 3 + .../src/platform_browser/browser_platform.ts | 4 + .../firestore/test/unit/util/bundle.test.ts | 297 +++++++++--------- 3 files changed, 157 insertions(+), 147 deletions(-) diff --git a/packages/firestore/src/platform/platform.ts b/packages/firestore/src/platform/platform.ts index f5c7e013713..af830a468bd 100644 --- a/packages/firestore/src/platform/platform.ts +++ b/packages/firestore/src/platform/platform.ts @@ -64,6 +64,7 @@ export interface Platform { export interface ByteStreamReader { read(): Promise; + cancel(reason?: string): Promise; } export interface ByteStreamReadResult { @@ -89,6 +90,8 @@ export function toByteStreamReader( return { value: new Uint8Array(), done: true }; } + + async cancel(reason?: string): Promise {} })(); } diff --git a/packages/firestore/src/platform_browser/browser_platform.ts b/packages/firestore/src/platform_browser/browser_platform.ts index 54be8b25cf5..ff1ab177568 100644 --- a/packages/firestore/src/platform_browser/browser_platform.ts +++ b/packages/firestore/src/platform_browser/browser_platform.ts @@ -112,6 +112,10 @@ export class BrowserPlatform implements Platform { read(): Promise { return reader.read(); } + + cancel(reason?: string): Promise { + return reader.cancel(reason); + } })(); } throw new Error( diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index 6e40459139a..d68b739f77e 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -17,22 +17,32 @@ import { expect } from 'chai'; import { BundleReader, - SizedBundleElement, - toReadableStream + SizedBundleElement } from '../../../src/util/bundle_reader'; import { isNode } from '../../util/test_platform'; +/** + * Create a `ReadableStream` from a underlying buffer. + * + * @param data: Underlying buffer. + * @param bytesPerRead: How many bytes to read from the underlying buffer from each read through the stream. + */ function readableStreamFromString( content: string, bytesPerRead: number -): ReadableStream { - return toReadableStream(new TextEncoder().encode(content), bytesPerRead); -} - -function lengthPrefixedString(o: {}): string { - const str = JSON.stringify(o); - const l = new TextEncoder().encode(str).byteLength; - return `${l}${str}`; +): ReadableStream { + const data = new TextEncoder().encode(content); + let readFrom = 0; + return new ReadableStream({ + start(controller) {}, + async pull(controller): Promise { + controller.enqueue(data.slice(readFrom, readFrom + bytesPerRead)); + readFrom += bytesPerRead; + if (readFrom >= data.byteLength) { + controller.close(); + } + } + }); } // eslint-disable-next-line no-restricted-properties @@ -60,16 +70,82 @@ function lengthPrefixedString(o: {}): string { }); }); -// eslint-disable-next-line no-restricted-properties -(isNode() ? describe.skip : describe)('Bundle ', () => { - genericBundleReadingTests(1); - genericBundleReadingTests(4); - genericBundleReadingTests(64); - genericBundleReadingTests(1024); +describe.only('Bundle ', () => { + if (!isNode()) { + genericBundleReadingTests(1); + genericBundleReadingTests(4); + genericBundleReadingTests(64); + genericBundleReadingTests(1024); + } + genericBundleReadingTests(0); }); function genericBundleReadingTests(bytesPerRead: number): void { const encoder = new TextEncoder(); + + function testTextSuffix(): string { + if (bytesPerRead > 0) { + return ` from ReadableStream with bytesPerRead: ${bytesPerRead}`; + } + return ' from Uint8Array'; + } + + function bundleFromString(s: string): BundleReader { + if (bytesPerRead > 0) { + return new BundleReader(readableStreamFromString(s, bytesPerRead)); + } + return new BundleReader(encoder.encode(s)); + } + + function lengthPrefixedString(o: {}): string { + const str = JSON.stringify(o); + const l = new TextEncoder().encode(str).byteLength; + return `${l}${str}`; + } + + async function parseThroughBundle( + bundleString: string, + validMeta = false + ): Promise { + const bundle = bundleFromString(bundleString); + + if (!validMeta) { + await expect(await bundle.getMetadata()).should.be.rejected; + } else { + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + } + + await getAllElement(bundle); + } + + async function getAllElement( + bundle: BundleReader + ): Promise { + const result: SizedBundleElement[] = []; + while (true) { + const sizedElement = await bundle.nextElement(); + if (sizedElement === null) { + break; + } + if (!sizedElement.isBundleMetadata()) { + result.push(sizedElement); + } + } + + return Promise.resolve(result); + } + + function verifySizedElement( + element: SizedBundleElement, + payload: unknown, + payloadString: string + ): void { + expect(element.payload).to.deep.equal(payload); + expect(element.byteLength).to.equal( + encoder.encode(payloadString).byteLength + ); + } + // Setting up test data. const meta = { metadata: { @@ -165,61 +241,14 @@ function genericBundleReadingTests(bytesPerRead: number): void { }; const limitToLastQueryString = lengthPrefixedString(limitToLastQuery); - async function getAllElement( - bundle: BundleReader - ): Promise { - const result: SizedBundleElement[] = []; - while (true) { - const sizedElement = await bundle.nextElement(); - if (sizedElement === null) { - break; - } - if (!sizedElement.isBundleMetadata()) { - result.push(sizedElement); - } - } - - return Promise.resolve(result); - } - - function verifySizedElement( - element: SizedBundleElement, - payload: unknown, - payloadString: string - ): void { - expect(element.payload).to.deep.equal(payload); - expect(element.byteLength).to.equal( - encoder.encode(payloadString).byteLength - ); - } - - async function parseThroughBundle( - bundleString: string, - bytesPerRead: number, - validMeta = false - ): Promise { - const bundleStream = readableStreamFromString(bundleString, bytesPerRead); - const bundle = new BundleReader(bundleStream); - - if (!validMeta) { - await expect(await bundle.getMetadata()).should.be.rejected; - } else { - expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); - } - - await getAllElement(bundle); - } - - it('reads with query and doc with bytesPerRead ' + bytesPerRead, async () => { - const bundleStream = readableStreamFromString( + it('reads with query and doc' + testTextSuffix(), async () => { + const bundle = bundleFromString( metaString + limitQueryString + limitToLastQueryString + doc1MetaString + - doc1String, - bytesPerRead + doc1String ); - const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); @@ -231,57 +260,43 @@ function genericBundleReadingTests(bytesPerRead: number): void { verifySizedElement(actual[3], doc1, doc1String); }); - it( - 'reads with unexpected orders with bytesPerRead ' + bytesPerRead, - async () => { - const bundleStream = readableStreamFromString( - metaString + - doc1MetaString + - doc1String + - limitQueryString + - doc2MetaString + - doc2String, - bytesPerRead - ); - const bundle = new BundleReader(bundleStream); - - const actual = await getAllElement(bundle); - expect(actual.length).to.equal(5); - verifySizedElement(actual[0], doc1Meta, doc1MetaString); - verifySizedElement(actual[1], doc1, doc1String); - verifySizedElement(actual[2], limitQuery, limitQueryString); - verifySizedElement(actual[3], doc2Meta, doc2MetaString); - verifySizedElement(actual[4], doc2, doc2String); - - // Reading metadata after other elements should also work. - expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); - } - ); + it('reads with unexpected orders' + testTextSuffix(), async () => { + const bundle = bundleFromString( + metaString + + doc1MetaString + + doc1String + + limitQueryString + + doc2MetaString + + doc2String + ); - it( - 'reads without named query with bytesPerRead ' + bytesPerRead, - async () => { - const bundleStream = readableStreamFromString( - metaString + doc1MetaString + doc1String, - bytesPerRead - ); - const bundle = new BundleReader(bundleStream); + const actual = await getAllElement(bundle); + expect(actual.length).to.equal(5); + verifySizedElement(actual[0], doc1Meta, doc1MetaString); + verifySizedElement(actual[1], doc1, doc1String); + verifySizedElement(actual[2], limitQuery, limitQueryString); + verifySizedElement(actual[3], doc2Meta, doc2MetaString); + verifySizedElement(actual[4], doc2, doc2String); + + // Reading metadata after other elements should also work. + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + }); - expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + it('reads without named query' + testTextSuffix(), async () => { + const bundle = bundleFromString(metaString + doc1MetaString + doc1String); - const actual = await getAllElement(bundle); - expect(actual.length).to.equal(2); - verifySizedElement(actual[0], doc1Meta, doc1MetaString); - verifySizedElement(actual[1], doc1, doc1String); - } - ); + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + + const actual = await getAllElement(bundle); + expect(actual.length).to.equal(2); + verifySizedElement(actual[0], doc1Meta, doc1MetaString); + verifySizedElement(actual[1], doc1, doc1String); + }); - it('reads with deleted doc with bytesPerRead ' + bytesPerRead, async () => { - const bundleStream = readableStreamFromString( - metaString + noDocMetaString + doc1MetaString + doc1String, - bytesPerRead + it('reads with deleted doc' + testTextSuffix(), async () => { + const bundle = bundleFromString( + metaString + noDocMetaString + doc1MetaString + doc1String ); - const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); @@ -292,41 +307,29 @@ function genericBundleReadingTests(bytesPerRead: number): void { verifySizedElement(actual[2], doc1, doc1String); }); - it( - 'reads without documents or query with bytesPerRead ' + bytesPerRead, - async () => { - const bundleStream = readableStreamFromString(metaString, bytesPerRead); - const bundle = new BundleReader(bundleStream); + it('reads without documents or query' + testTextSuffix(), async () => { + const bundle = bundleFromString(metaString); - expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); + expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); - const actual = await getAllElement(bundle); - expect(actual.length).to.equal(0); - } - ); - - it( - 'throws with ill-formatted bundle with bytesPerRead ' + bytesPerRead, - async () => { - await expect( - parseThroughBundle('metadata: "no length prefix"', bytesPerRead) - ).to.be.rejected; - - await expect( - parseThroughBundle('{metadata: "no length prefix"}', bytesPerRead) - ).to.be.rejected; - - await expect( - parseThroughBundle(metaString + 'invalid-string', bytesPerRead, true) - ).to.be.rejected; - - await expect(parseThroughBundle('1' + metaString, bytesPerRead)).to.be - .rejected; - - // First element is not BundleMetadata. - await expect( - parseThroughBundle(doc1MetaString + doc1String, bytesPerRead) - ).to.be.rejected; - } - ); + const actual = await getAllElement(bundle); + expect(actual.length).to.equal(0); + }); + + it('throws with ill-formatted bundle' + testTextSuffix(), async () => { + await expect(parseThroughBundle('metadata: "no length prefix"')).to.be + .rejected; + + await expect(parseThroughBundle('{metadata: "no length prefix"}')).to.be + .rejected; + + await expect(parseThroughBundle(metaString + 'invalid-string', true)).to.be + .rejected; + + await expect(parseThroughBundle('1' + metaString)).to.be.rejected; + + // First element is not BundleMetadata. + await expect(parseThroughBundle(doc1MetaString + doc1String)).to.be + .rejected; + }); } From 24e10cb6e06f68c1e8e61859c4fe55c52778cc7f Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 29 May 2020 14:38:49 -0400 Subject: [PATCH 09/39] Remote 'only' --- packages/firestore/test/unit/util/bundle.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index d68b739f77e..ceeae8b2d76 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -70,7 +70,7 @@ function readableStreamFromString( }); }); -describe.only('Bundle ', () => { +describe('Bundle ', () => { if (!isNode()) { genericBundleReadingTests(1); genericBundleReadingTests(4); From 4cbe60899d2f19df04a9f5b0f0febd67273bbdd8 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 4 Jun 2020 22:16:35 -0400 Subject: [PATCH 10/39] Merge branch 'wuandy/Bundles' into wuandy/BundleReaderNode # Conflicts: # packages/firestore/src/util/bundle_reader.ts # packages/firestore/test/unit/util/bundle.test.ts --- packages/firestore/src/platform/platform.ts | 4 + packages/firestore/src/util/bundle_reader.ts | 39 ++------- .../firestore/test/unit/util/bundle.test.ts | 83 ++++++++++++------- 3 files changed, 64 insertions(+), 62 deletions(-) diff --git a/packages/firestore/src/platform/platform.ts b/packages/firestore/src/platform/platform.ts index af830a468bd..034e5a5e64b 100644 --- a/packages/firestore/src/platform/platform.ts +++ b/packages/firestore/src/platform/platform.ts @@ -109,6 +109,10 @@ export class PlatformSupport { PlatformSupport.platform = platform; } + private static _forceSetPlatform(platform: Platform): void { + PlatformSupport.platform = platform; + } + static getPlatform(): Platform { if (!PlatformSupport.platform) { fail('Platform not set'); diff --git a/packages/firestore/src/util/bundle_reader.ts b/packages/firestore/src/util/bundle_reader.ts index c8e7435953b..c608357a293 100644 --- a/packages/firestore/src/util/bundle_reader.ts +++ b/packages/firestore/src/util/bundle_reader.ts @@ -20,6 +20,7 @@ import { BundleMetadata } from '../protos/firestore_bundle_proto'; import { Deferred } from './promise'; +import { ByteStreamReader, PlatformSupport } from '../platform/platform'; /** * A complete element in the bundle stream, together with the byte length it @@ -37,30 +38,6 @@ export class SizedBundleElement { } } -/** - * Create a `ReadableStream` from a underlying buffer. - * - * @param data: Underlying buffer. - * @param bytesPerRead: How many bytes to read from the underlying buffer from - * each read through the stream. - */ -export function toReadableStream( - data: Uint8Array | ArrayBuffer, - bytesPerRead = 10240 -): ReadableStream { - let readFrom = 0; - return new ReadableStream({ - start(controller) {}, - async pull(controller): Promise { - controller.enqueue(data.slice(readFrom, readFrom + bytesPerRead)); - readFrom += bytesPerRead; - if (readFrom >= data.byteLength) { - controller.close(); - } - } - }); -} - /** * A class representing a bundle. * @@ -70,8 +47,8 @@ export function toReadableStream( export class BundleReader { /** Cached bundle metadata. */ private metadata: Deferred = new Deferred(); - /** The reader instance of the given ReadableStream. */ - private reader: ReadableStreamDefaultReader; + /** The reader to read from underlying binary bundle data source. */ + private reader: ByteStreamReader; /** * Internal buffer to hold bundle content, accumulating incomplete element * content. @@ -86,13 +63,9 @@ export class BundleReader { | Uint8Array | ArrayBuffer ) { - if ( - bundleStream instanceof Uint8Array || - bundleStream instanceof ArrayBuffer - ) { - this.bundleStream = toReadableStream(bundleStream); - } - this.reader = (this.bundleStream as ReadableStream).getReader(); + this.reader = PlatformSupport.getPlatform().toByteStreamReader( + bundleStream + ); // Read the metadata (which is the first element). this.nextElementImpl().then( diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index f163c29cbb5..42841c02746 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -17,17 +17,38 @@ import { expect } from 'chai'; import { BundleReader, - SizedBundleElement, - toReadableStream + SizedBundleElement } from '../../../src/util/bundle_reader'; -import { isNode } from '../../util/test_platform'; import { BundleElement } from '../../../src/protos/firestore_bundle_proto'; +import { isNode } from '../../util/test_platform'; +import { + PlatformSupport, + toByteStreamReader +} from '../../../src/platform/platform'; -function readableStreamFromString( +/** + * Create a `ReadableStream` from a string. + * + * @param content: Bundle in string. + * @param bytesPerRead: How many bytes to read from the underlying buffer from + * each read through the stream. + */ +export function readableStreamFromString( content: string, - bytesPerRead: number -): ReadableStream { - return toReadableStream(new TextEncoder().encode(content), bytesPerRead); + bytesPerRead = 10240 +): ReadableStream { + let readFrom = 0; + const data = new TextEncoder().encode(content); + return new ReadableStream({ + start(controller) {}, + async pull(controller): Promise { + controller.enqueue(data.slice(readFrom, readFrom + bytesPerRead)); + readFrom += bytesPerRead; + if (readFrom >= data.byteLength) { + controller.close(); + } + } + }); } function lengthPrefixedString(o: {}): string { @@ -60,7 +81,7 @@ describe('readableStreamFromString()', () => { }); }); -describe('Bundle ', () => { +describe.only('Bundle ', () => { genericBundleReadingTests(1); genericBundleReadingTests(4); genericBundleReadingTests(64); @@ -68,6 +89,22 @@ describe('Bundle ', () => { }); function genericBundleReadingTests(bytesPerRead: number): void { + if (isNode()) { + const platform = PlatformSupport.getPlatform(); + platform.toByteStreamReader = source => + toByteStreamReader(source as Uint8Array, bytesPerRead); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (PlatformSupport as any)._forceSetPlatform(platform); + } + + function bundleFromString(s: string): BundleReader { + if (isNode()) { + return new BundleReader(encoder.encode(s)); + } else { + return new BundleReader(readableStreamFromString(s, bytesPerRead)); + } + } + const encoder = new TextEncoder(); // Setting up test data. const meta: BundleElement = { @@ -197,8 +234,7 @@ function genericBundleReadingTests(bytesPerRead: number): void { bytesPerRead: number, validMeta = false ): Promise { - const bundleStream = readableStreamFromString(bundleString, bytesPerRead); - const bundle = new BundleReader(bundleStream); + const bundle = bundleFromString(bundleString); if (!validMeta) { await expect(await bundle.getMetadata()).should.be.rejected; @@ -210,15 +246,13 @@ function genericBundleReadingTests(bytesPerRead: number): void { } it('reads with query and doc with bytesPerRead ' + bytesPerRead, async () => { - const bundleStream = readableStreamFromString( + const bundle = bundleFromString( metaString + limitQueryString + limitToLastQueryString + doc1MetaString + - doc1String, - bytesPerRead + doc1String ); - const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); @@ -233,16 +267,14 @@ function genericBundleReadingTests(bytesPerRead: number): void { it( 'reads with unexpected orders with bytesPerRead ' + bytesPerRead, async () => { - const bundleStream = readableStreamFromString( + const bundle = bundleFromString( metaString + doc1MetaString + doc1String + limitQueryString + doc2MetaString + - doc2String, - bytesPerRead + doc2String ); - const bundle = new BundleReader(bundleStream); const actual = await getAllElements(bundle); expect(actual.length).to.equal(5); @@ -260,11 +292,7 @@ function genericBundleReadingTests(bytesPerRead: number): void { it( 'reads without named query with bytesPerRead ' + bytesPerRead, async () => { - const bundleStream = readableStreamFromString( - metaString + doc1MetaString + doc1String, - bytesPerRead - ); - const bundle = new BundleReader(bundleStream); + const bundle = bundleFromString(metaString + doc1MetaString + doc1String); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); @@ -276,11 +304,9 @@ function genericBundleReadingTests(bytesPerRead: number): void { ); it('reads with deleted doc with bytesPerRead ' + bytesPerRead, async () => { - const bundleStream = readableStreamFromString( - metaString + noDocMetaString + doc1MetaString + doc1String, - bytesPerRead + const bundle = bundleFromString( + metaString + noDocMetaString + doc1MetaString + doc1String ); - const bundle = new BundleReader(bundleStream); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); @@ -294,8 +320,7 @@ function genericBundleReadingTests(bytesPerRead: number): void { it( 'reads without documents or query with bytesPerRead ' + bytesPerRead, async () => { - const bundleStream = readableStreamFromString(metaString, bytesPerRead); - const bundle = new BundleReader(bundleStream); + const bundle = bundleFromString(metaString); expect(await bundle.getMetadata()).to.deep.equal(meta.metadata); From 4313e51b4da4d04528764fe22965eced633b2764 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 5 Jun 2020 10:58:46 -0400 Subject: [PATCH 11/39] Added more comments, and more tests for Node. --- packages/firestore/src/platform/platform.ts | 25 ++++++++++++++++-- .../src/platform_browser/browser_platform.ts | 7 ++++- .../src/platform_node/node_platform.ts | 3 +++ packages/firestore/src/util/bundle_reader.ts | 6 +++-- .../test/unit/platform/platform.test.ts | 26 ++++++++++++++++++- .../firestore/test/unit/util/bundle.test.ts | 12 ++++++--- 6 files changed, 69 insertions(+), 10 deletions(-) diff --git a/packages/firestore/src/platform/platform.ts b/packages/firestore/src/platform/platform.ts index 034e5a5e64b..4bc25e25279 100644 --- a/packages/firestore/src/platform/platform.ts +++ b/packages/firestore/src/platform/platform.ts @@ -50,6 +50,9 @@ export interface Platform { */ randomBytes(nBytes: number): Uint8Array; + /** + * Builds a `ByteStreamReader` from a data source. + */ toByteStreamReader(source: unknown): ByteStreamReader; /** The Platform's 'window' implementation or null if not available. */ @@ -62,16 +65,31 @@ export interface Platform { readonly base64Available: boolean; } +/** + * An interface compatible with Web's ReadableStream.getReader() return type. + * + * This can be used as an abstraction to mimic `ReadableStream` where it is not + * available. + */ export interface ByteStreamReader { read(): Promise; cancel(reason?: string): Promise; } +/** + * An interface compatible with ReadableStreamReadResult. + */ export interface ByteStreamReadResult { done: boolean; - value: Uint8Array; + value?: Uint8Array; } +/** + * Builds a `ByteStreamReader` from a UInt8Array. + * @param source The data source to use. + * @param bytesPerRead How many bytes each `read()` from the returned reader + * will read. + */ export function toByteStreamReader( source: Uint8Array, bytesPerRead = 10240 @@ -88,7 +106,7 @@ export function toByteStreamReader( return result; } - return { value: new Uint8Array(), done: true }; + return { value: undefined, done: true }; } async cancel(reason?: string): Promise {} @@ -109,6 +127,9 @@ export class PlatformSupport { PlatformSupport.platform = platform; } + /** + * Forcing to set the platform instance, testing only! + */ private static _forceSetPlatform(platform: Platform): void { PlatformSupport.platform = platform; } diff --git a/packages/firestore/src/platform_browser/browser_platform.ts b/packages/firestore/src/platform_browser/browser_platform.ts index ff1ab177568..9941bcf4e7c 100644 --- a/packages/firestore/src/platform_browser/browser_platform.ts +++ b/packages/firestore/src/platform_browser/browser_platform.ts @@ -99,7 +99,12 @@ export class BrowserPlatform implements Platform { return bytes; } - toByteStreamReader(source: unknown): ByteStreamReader { + /** + * On web, a `ReadableStream` is wrapped around by a `ByteStreamReader`. + */ + toByteStreamReader( + source: Uint8Array | ArrayBuffer | ReadableStream + ): ByteStreamReader { if (source instanceof Uint8Array) { return toByteStreamReader(source); } diff --git a/packages/firestore/src/platform_node/node_platform.ts b/packages/firestore/src/platform_node/node_platform.ts index c2a40b76aa9..575de6616c2 100644 --- a/packages/firestore/src/platform_node/node_platform.ts +++ b/packages/firestore/src/platform_node/node_platform.ts @@ -88,6 +88,9 @@ export class NodePlatform implements Platform { return randomBytes(nBytes); } + /** + * On Node, only supported data source is a `Uint8Array` for now. + */ toByteStreamReader(source: unknown): ByteStreamReader { if (source instanceof Uint8Array) { return toByteStreamReader(source); diff --git a/packages/firestore/src/util/bundle_reader.ts b/packages/firestore/src/util/bundle_reader.ts index c608357a293..aff7a25c77d 100644 --- a/packages/firestore/src/util/bundle_reader.ts +++ b/packages/firestore/src/util/bundle_reader.ts @@ -21,6 +21,7 @@ import { } from '../protos/firestore_bundle_proto'; import { Deferred } from './promise'; import { ByteStreamReader, PlatformSupport } from '../platform/platform'; +import { debugAssert } from './assert'; /** * A complete element in the bundle stream, together with the byte length it @@ -204,11 +205,12 @@ export class BundleReader { private async pullMoreDataToBuffer(): Promise { const result = await this.reader.read(); if (!result.done) { + debugAssert(!!result.value, 'Read undefined when "done" is false.'); const newBuffer = new Uint8Array( - this.buffer.length + result.value.length + this.buffer.length + result.value!.length ); newBuffer.set(this.buffer); - newBuffer.set(result.value, this.buffer.length); + newBuffer.set(result.value!, this.buffer.length); this.buffer = newBuffer; } return result.done; diff --git a/packages/firestore/test/unit/platform/platform.test.ts b/packages/firestore/test/unit/platform/platform.test.ts index 289b0edee0d..631c49fb23d 100644 --- a/packages/firestore/test/unit/platform/platform.test.ts +++ b/packages/firestore/test/unit/platform/platform.test.ts @@ -16,10 +16,34 @@ */ import { expect } from 'chai'; -import { PlatformSupport } from '../../../src/platform/platform'; +import { + PlatformSupport, + toByteStreamReader +} from '../../../src/platform/platform'; describe('Platform', () => { it('can load the platform at runtime', () => { expect(PlatformSupport.getPlatform()).to.exist; }); + + it('toByteStreamReader() steps underlying data', async () => { + const encoder = new TextEncoder(); + const r = toByteStreamReader(encoder.encode('0123456789'), 4); + + let result = await r.read(); + expect(result.value).to.deep.equal(encoder.encode('0123')); + expect(result.done).to.be.false; + + result = await r.read(); + expect(result.value).to.deep.equal(encoder.encode('4567')); + expect(result.done).to.be.false; + + result = await r.read(); + expect(result.value).to.deep.equal(encoder.encode('89')); + expect(result.done).to.be.false; + + result = await r.read(); + expect(result.value).to.be.undefined; + expect(result.done).to.be.true; + }); }); diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index 42841c02746..e915d2e5dfc 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -57,11 +57,14 @@ function lengthPrefixedString(o: {}): string { return `${l}${str}`; } -describe('readableStreamFromString()', () => { +// Testing readableStreamFromString() is working as expected. +// eslint-disable-next-line no-restricted-properties +(isNode() ? describe.skip : describe)('readableStreamFromString()', () => { it('returns stepping readable stream', async () => { const encoder = new TextEncoder(); - const s = readableStreamFromString('0123456789', 4); - const r = s.getReader(); + const r = PlatformSupport.getPlatform().toByteStreamReader( + readableStreamFromString('0123456789', 4) + ); let result = await r.read(); expect(result.value).to.deep.equal(encoder.encode('0123')); @@ -81,7 +84,7 @@ describe('readableStreamFromString()', () => { }); }); -describe.only('Bundle ', () => { +describe('Bundle ', () => { genericBundleReadingTests(1); genericBundleReadingTests(4); genericBundleReadingTests(64); @@ -89,6 +92,7 @@ describe.only('Bundle ', () => { }); function genericBundleReadingTests(bytesPerRead: number): void { + // On Node, we need to override `bytesPerRead` from it's platform's `toByteStreamReader` call. if (isNode()) { const platform = PlatformSupport.getPlatform(); platform.toByteStreamReader = source => From 296cfc4300347eb08adc1e232fd792f8370fc462 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 5 Jun 2020 11:15:01 -0400 Subject: [PATCH 12/39] Implement BundleCache. --- packages/firestore/src/core/bundle.ts | 39 ++++ .../firestore/src/core/component_provider.ts | 3 +- packages/firestore/src/local/bundle_cache.ts | 60 +++++ .../src/local/indexeddb_bundle_cache.ts | 105 +++++++++ .../src/local/indexeddb_persistence.ts | 11 + .../firestore/src/local/indexeddb_schema.ts | 18 +- .../firestore/src/local/local_serializer.ts | 80 +++++++ .../src/local/memory_bundle_cache.ts | 66 ++++++ .../firestore/src/local/memory_persistence.ts | 14 +- packages/firestore/src/local/persistence.ts | 10 + packages/firestore/src/remote/serializer.ts | 8 +- .../test/unit/local/bundle_cache.test.ts | 207 ++++++++++++++++++ .../unit/local/persistence_test_helpers.ts | 9 +- .../test/unit/local/test_bundle_cache.ts | 73 ++++++ .../test/unit/specs/spec_test_components.ts | 6 +- 15 files changed, 692 insertions(+), 17 deletions(-) create mode 100644 packages/firestore/src/core/bundle.ts create mode 100644 packages/firestore/src/local/bundle_cache.ts create mode 100644 packages/firestore/src/local/indexeddb_bundle_cache.ts create mode 100644 packages/firestore/src/local/memory_bundle_cache.ts create mode 100644 packages/firestore/test/unit/local/bundle_cache.test.ts create mode 100644 packages/firestore/test/unit/local/test_bundle_cache.ts diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts new file mode 100644 index 00000000000..0571664b7f4 --- /dev/null +++ b/packages/firestore/src/core/bundle.ts @@ -0,0 +1,39 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Query } from './query'; +import { SnapshotVersion } from './snapshot_version'; + +/** + * Represents a Firestore bundle saved by the SDK in its local storage. + */ +export interface Bundle { + readonly id: string; + readonly version: number; + // When the saved bundle is built from the server SDKs. + readonly createTime: SnapshotVersion; +} + +/** + * Represents a Query saved by the SDK in its local storage. + */ +export interface NamedQuery { + readonly name: string; + readonly query: Query; + // When the results for this query are read to the saved bundle. + readonly readTime: SnapshotVersion; +} diff --git a/packages/firestore/src/core/component_provider.ts b/packages/firestore/src/core/component_provider.ts index 2c34723ecef..38ba5d2c763 100644 --- a/packages/firestore/src/core/component_provider.ts +++ b/packages/firestore/src/core/component_provider.ts @@ -137,7 +137,8 @@ export class MemoryComponentProvider implements ComponentProvider { !cfg.persistenceSettings.durable, 'Can only start memory persistence' ); - return new MemoryPersistence(MemoryEagerDelegate.factory); + const serializer = cfg.platform.newSerializer(cfg.databaseInfo.databaseId); + return new MemoryPersistence(MemoryEagerDelegate.factory, serializer); } createRemoteStore(cfg: ComponentConfiguration): RemoteStore { diff --git a/packages/firestore/src/local/bundle_cache.ts b/packages/firestore/src/local/bundle_cache.ts new file mode 100644 index 00000000000..e24d6ebc52f --- /dev/null +++ b/packages/firestore/src/local/bundle_cache.ts @@ -0,0 +1,60 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PersistenceTransaction } from './persistence'; +import { PersistencePromise } from './persistence_promise'; +import * as bundleProto from '../protos/firestore_bundle_proto'; +import { Bundle, NamedQuery } from '../core/bundle'; + +/** + * Provides interfaces to save and read Firestore bundles. + */ +export interface BundleCache { + /** + * Gets a saved `Bundle` for a given `bundleId`, returns undefined if + * no bundles are found under the given id. + */ + getBundle( + transaction: PersistenceTransaction, + bundleId: string + ): PersistencePromise; + + /** + * Saves a `BundleMetadata` from a bundle into local storage, using its id as the persistent key. + */ + saveBundleMetadata( + transaction: PersistenceTransaction, + metadata: bundleProto.BundleMetadata + ): PersistencePromise; + + /** + * Gets a saved `NamedQuery` for the given query name. Returns undefined if + * no queries are found under the given name. + */ + getNamedQuery( + transaction: PersistenceTransaction, + queryName: string + ): PersistencePromise; + + /** + * Saves a `NamedQuery` from a bundle, using its name as the persistent key. + */ + saveNamedQuery( + transaction: PersistenceTransaction, + query: bundleProto.NamedQuery + ): PersistencePromise; +} diff --git a/packages/firestore/src/local/indexeddb_bundle_cache.ts b/packages/firestore/src/local/indexeddb_bundle_cache.ts new file mode 100644 index 00000000000..a23546e3d4b --- /dev/null +++ b/packages/firestore/src/local/indexeddb_bundle_cache.ts @@ -0,0 +1,105 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PersistenceTransaction } from './persistence'; +import { PersistencePromise } from './persistence_promise'; +import * as bundleProto from '../protos/firestore_bundle_proto'; +import { BundleCache } from './bundle_cache'; +import { + DbBundle, + DbBundlesKey, + DbNamedQuery, + DbNamedQueriesKey +} from './indexeddb_schema'; +import { SimpleDbStore } from './simple_db'; +import { IndexedDbPersistence } from './indexeddb_persistence'; +import { LocalSerializer } from './local_serializer'; +import { Bundle, NamedQuery } from '../core/bundle'; + +export class IndexedDbBundleCache implements BundleCache { + constructor(private serializer: LocalSerializer) {} + + getBundle( + transaction: PersistenceTransaction, + bundleId: string + ): PersistencePromise { + return bundlesStore(transaction) + .get(bundleId) + .next(bundle => { + if (bundle) { + return this.serializer.fromDbBundle(bundle!); + } + return undefined; + }); + } + + saveBundleMetadata( + transaction: PersistenceTransaction, + bundleMetadata: bundleProto.BundleMetadata + ): PersistencePromise { + return bundlesStore(transaction).put( + this.serializer.toDbBundle(bundleMetadata) + ); + } + + getNamedQuery( + transaction: PersistenceTransaction, + queryName: string + ): PersistencePromise { + return namedQueriesStore(transaction) + .get(queryName) + .next(query => { + if (query) { + return this.serializer.fromDbNamedQuery(query!); + } + return undefined; + }); + } + + saveNamedQuery( + transaction: PersistenceTransaction, + query: bundleProto.NamedQuery + ): PersistencePromise { + return namedQueriesStore(transaction).put( + this.serializer.toDbNamedQuery(query) + ); + } +} + +/** + * Helper to get a typed SimpleDbStore for the bundles object store. + */ +function bundlesStore( + txn: PersistenceTransaction +): SimpleDbStore { + return IndexedDbPersistence.getStore( + txn, + DbBundle.store + ); +} + +/** + * Helper to get a typed SimpleDbStore for the namedQueries object store. + */ +function namedQueriesStore( + txn: PersistenceTransaction +): SimpleDbStore { + return IndexedDbPersistence.getStore( + txn, + DbNamedQuery.store + ); +} diff --git a/packages/firestore/src/local/indexeddb_persistence.ts b/packages/firestore/src/local/indexeddb_persistence.ts index 4409e0fab79..86d183f8f2e 100644 --- a/packages/firestore/src/local/indexeddb_persistence.ts +++ b/packages/firestore/src/local/indexeddb_persistence.ts @@ -32,6 +32,7 @@ import { EncodedResourcePath, encodeResourcePath } from './encoded_resource_path'; +import { IndexedDbBundleCache } from './indexeddb_bundle_cache'; import { IndexedDbIndexManager } from './indexeddb_index_manager'; import { IndexedDbMutationQueue, @@ -226,6 +227,7 @@ export class IndexedDbPersistence implements Persistence { private readonly targetCache: IndexedDbTargetCache; private readonly indexManager: IndexedDbIndexManager; private readonly remoteDocumentCache: IndexedDbRemoteDocumentCache; + private readonly bundleCache: IndexedDbBundleCache; private readonly webStorage: Storage; readonly referenceDelegate: IndexedDbLruDelegate; @@ -259,6 +261,7 @@ export class IndexedDbPersistence implements Persistence { this.serializer, this.indexManager ); + this.bundleCache = new IndexedDbBundleCache(this.serializer); if (platform.window && platform.window.localStorage) { this.window = platform.window; this.webStorage = this.window.localStorage; @@ -763,6 +766,14 @@ export class IndexedDbPersistence implements Persistence { return this.indexManager; } + getBundleCache(): IndexedDbBundleCache { + debugAssert( + this.started, + 'Cannot initialize BundleCache before persistence is started.' + ); + return this.bundleCache; + } + runTransaction( action: string, mode: PersistenceTransactionMode, diff --git a/packages/firestore/src/local/indexeddb_schema.ts b/packages/firestore/src/local/indexeddb_schema.ts index 2026b373adb..579a8d9eeeb 100644 --- a/packages/firestore/src/local/indexeddb_schema.ts +++ b/packages/firestore/src/local/indexeddb_schema.ts @@ -1092,11 +1092,11 @@ export type DbBundlesKey = string; /** * A object representing a bundle loaded by the SDK. */ -export class DbBundles { +export class DbBundle { /** Name of the IndexedDb object store. */ static store = 'bundles'; - static keyPath = ['bundleId']; + static keyPath = 'bundleId'; constructor( /** The ID of the loaded bundle. */ @@ -1109,8 +1109,8 @@ export class DbBundles { } function createBundlesStore(db: IDBDatabase): void { - db.createObjectStore(DbBundles.store, { - keyPath: DbBundles.keyPath + db.createObjectStore(DbBundle.store, { + keyPath: DbBundle.keyPath }); } @@ -1119,11 +1119,11 @@ export type DbNamedQueriesKey = string; /** * A object representing a named query loaded by the SDK via a bundle. */ -export class DbNamedQueries { +export class DbNamedQuery { /** Name of the IndexedDb object store. */ static store = 'namedQueries'; - static keyPath = ['name']; + static keyPath = 'name'; constructor( /** The name of the query. */ @@ -1136,8 +1136,8 @@ export class DbNamedQueries { } function createNamedQueriesStore(db: IDBDatabase): void { - db.createObjectStore(DbNamedQueries.store, { - keyPath: DbNamedQueries.keyPath + db.createObjectStore(DbNamedQuery.store, { + keyPath: DbNamedQuery.keyPath }); } @@ -1174,7 +1174,7 @@ export const V8_STORES = [...V6_STORES, DbCollectionParent.store]; // V10 does not change the set of stores. -export const V11_STORES = [...V8_STORES, DbCollectionParent.store]; +export const V11_STORES = [...V8_STORES, DbBundle.store, DbNamedQuery.store]; /** * The list of all default IndexedDB stores used throughout the SDK. This is diff --git a/packages/firestore/src/local/local_serializer.ts b/packages/firestore/src/local/local_serializer.ts index e15697b934c..a6176ceb2e1 100644 --- a/packages/firestore/src/local/local_serializer.ts +++ b/packages/firestore/src/local/local_serializer.ts @@ -25,13 +25,16 @@ import { } from '../model/document'; import { DocumentKey } from '../model/document_key'; import { MutationBatch } from '../model/mutation_batch'; +import * as bundleProto from '../protos/firestore_bundle_proto'; import * as api from '../protos/firestore_proto_api'; import { JsonProtoSerializer } from '../remote/serializer'; import { debugAssert, fail } from '../util/assert'; import { ByteString } from '../util/byte_string'; import { Target } from '../core/target'; import { + DbBundle, DbMutationBatch, + DbNamedQuery, DbNoDocument, DbQuery, DbRemoteDocument, @@ -41,6 +44,8 @@ import { DbUnknownDocument } from './indexeddb_schema'; import { TargetData, TargetPurpose } from './target_data'; +import { Bundle, NamedQuery } from '../core/bundle'; +import { Query } from '../core/query'; /** Serializer for values stored in the LocalStore. */ export class LocalSerializer { @@ -231,6 +236,81 @@ export class LocalSerializer { queryProto ); } + + /** Encodes a DbBundle to a Bundle. */ + fromDbBundle(dbBundle: DbBundle): Bundle { + return { + id: dbBundle.bundleId, + createTime: this.fromDbTimestamp(dbBundle.createTime), + version: dbBundle.version + }; + } + + /** Encodes a BundleMetadata to a DbBundle. */ + toDbBundle(metadata: bundleProto.BundleMetadata): DbBundle { + return { + bundleId: metadata.id!, + createTime: this.toDbTimestamp( + this.remoteSerializer.fromVersion(metadata.createTime!) + ), + version: metadata.version! + }; + } + + /** Encodes a DbNamedQuery to a NamedQuery. */ + fromDbNamedQuery(dbNamedQuery: DbNamedQuery): NamedQuery { + return { + name: dbNamedQuery.name, + query: this.fromBundledQuery(dbNamedQuery.bundledQuery), + readTime: this.fromDbTimestamp(dbNamedQuery.readTime) + }; + } + + /** Encodes a NamedQuery from bundle proto to a DbNamedQuery. */ + toDbNamedQuery(query: bundleProto.NamedQuery): DbNamedQuery { + return { + name: query.name!, + readTime: this.toDbTimestamp( + this.remoteSerializer.fromVersion(query.readTime!) + ), + bundledQuery: query.bundledQuery! + }; + } + + /** + * Encodes a `BundledQuery` from bundle proto to a Query object. + * + * This reconstructs the original query used to build the bundle being loaded, + * including features exists only in SDKs (for example: limit-to-last). + */ + fromBundledQuery(bundledQuery: bundleProto.BundledQuery): Query { + const query = this.remoteSerializer.convertQueryTargetToQuery({ + parent: bundledQuery.parent!, + structuredQuery: bundledQuery.structuredQuery! + }); + if (bundledQuery.limitType === 'LAST') { + return query.withLimitToLast(query.limit); + } + return query; + } + + /** Encodes a NamedQuery proto object to a NamedQuery model object. */ + fromProtoNamedQuery(namedQuery: bundleProto.NamedQuery): NamedQuery { + return { + name: namedQuery.name!, + query: this.fromBundledQuery(namedQuery.bundledQuery!), + readTime: this.remoteSerializer.fromVersion(namedQuery.readTime!) + }; + } + + /** Encodes a BundleMetadata proto object to a Bundle model object. */ + fromBundleMetadata(metadata: bundleProto.BundleMetadata): Bundle { + return { + id: metadata.id!, + version: metadata.version!, + createTime: this.remoteSerializer.fromVersion(metadata.createTime!) + }; + } } /** diff --git a/packages/firestore/src/local/memory_bundle_cache.ts b/packages/firestore/src/local/memory_bundle_cache.ts new file mode 100644 index 00000000000..64cbd7ef726 --- /dev/null +++ b/packages/firestore/src/local/memory_bundle_cache.ts @@ -0,0 +1,66 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PersistenceTransaction } from './persistence'; +import { PersistencePromise } from './persistence_promise'; +import * as bundleProto from '../protos/firestore_bundle_proto'; +import { BundleCache } from './bundle_cache'; +import { Bundle, NamedQuery } from '../core/bundle'; +import { LocalSerializer } from './local_serializer'; + +export class MemoryBundleCache implements BundleCache { + private bundles = new Map(); + private namedQueries = new Map(); + + constructor(private serializer: LocalSerializer) {} + + getBundle( + transaction: PersistenceTransaction, + bundleId: string + ): PersistencePromise { + return PersistencePromise.resolve(this.bundles.get(bundleId)); + } + + saveBundleMetadata( + transaction: PersistenceTransaction, + bundleMetadata: bundleProto.BundleMetadata + ): PersistencePromise { + this.bundles.set( + bundleMetadata.id!, + this.serializer.fromBundleMetadata(bundleMetadata) + ); + return PersistencePromise.resolve(); + } + + getNamedQuery( + transaction: PersistenceTransaction, + queryName: string + ): PersistencePromise { + return PersistencePromise.resolve(this.namedQueries.get(queryName)); + } + + saveNamedQuery( + transaction: PersistenceTransaction, + query: bundleProto.NamedQuery + ): PersistencePromise { + this.namedQueries.set( + query.name!, + this.serializer.fromProtoNamedQuery(query) + ); + return PersistencePromise.resolve(); + } +} diff --git a/packages/firestore/src/local/memory_persistence.ts b/packages/firestore/src/local/memory_persistence.ts index 391f8686565..ce84ff95ff2 100644 --- a/packages/firestore/src/local/memory_persistence.ts +++ b/packages/firestore/src/local/memory_persistence.ts @@ -45,6 +45,9 @@ import { import { PersistencePromise } from './persistence_promise'; import { ReferenceSet } from './reference_set'; import { TargetData } from './target_data'; +import { MemoryBundleCache } from './memory_bundle_cache'; +import { JsonProtoSerializer } from '../remote/serializer'; +import { LocalSerializer } from './local_serializer'; const LOG_TAG = 'MemoryPersistence'; /** @@ -63,7 +66,9 @@ export class MemoryPersistence implements Persistence { private mutationQueues: { [user: string]: MemoryMutationQueue } = {}; private readonly remoteDocumentCache: MemoryRemoteDocumentCache; private readonly targetCache: MemoryTargetCache; + private readonly bundleCache: MemoryBundleCache; private readonly listenSequence = new ListenSequence(0); + private serializer: LocalSerializer; private _started = false; @@ -76,7 +81,8 @@ export class MemoryPersistence implements Persistence { * checked or asserted on every access. */ constructor( - referenceDelegateFactory: (p: MemoryPersistence) => MemoryReferenceDelegate + referenceDelegateFactory: (p: MemoryPersistence) => MemoryReferenceDelegate, + serializer: JsonProtoSerializer ) { this._started = true; this.referenceDelegate = referenceDelegateFactory(this); @@ -88,6 +94,8 @@ export class MemoryPersistence implements Persistence { this.indexManager, sizer ); + this.serializer = new LocalSerializer(serializer); + this.bundleCache = new MemoryBundleCache(this.serializer); } start(): Promise { @@ -132,6 +140,10 @@ export class MemoryPersistence implements Persistence { return this.remoteDocumentCache; } + getBundleCache(): MemoryBundleCache { + return this.bundleCache; + } + runTransaction( action: string, mode: PersistenceTransactionMode, diff --git a/packages/firestore/src/local/persistence.ts b/packages/firestore/src/local/persistence.ts index 146b0331738..84debbc8afa 100644 --- a/packages/firestore/src/local/persistence.ts +++ b/packages/firestore/src/local/persistence.ts @@ -25,6 +25,7 @@ import { PersistencePromise } from './persistence_promise'; import { TargetCache } from './target_cache'; import { RemoteDocumentCache } from './remote_document_cache'; import { TargetData } from './target_data'; +import { BundleCache } from './bundle_cache'; export const PRIMARY_LEASE_LOST_ERROR_MSG = 'The current tab is not in the required state to perform this operation. ' + @@ -216,6 +217,15 @@ export interface Persistence { */ getRemoteDocumentCache(): RemoteDocumentCache; + /** + * Returns a BundleCache representing the persisted cache of loaded bundles. + * + * Note: The implementation is free to return the same instance every time + * this is called. In particular, the memory-backed implementation does this + * to emulate the persisted implementation to the extent possible. + */ + getBundleCache(): BundleCache; + /** * Returns an IndexManager instance that manages our persisted query indexes. * diff --git a/packages/firestore/src/remote/serializer.ts b/packages/firestore/src/remote/serializer.ts index de29942ffc2..db1ff3f3c39 100644 --- a/packages/firestore/src/remote/serializer.ts +++ b/packages/firestore/src/remote/serializer.ts @@ -787,7 +787,7 @@ export class JsonProtoSerializer { return result; } - fromQueryTarget(target: api.QueryTarget): Target { + convertQueryTargetToQuery(target: api.QueryTarget): Query { let path = this.fromQueryPath(target.parent!); const query = target.structuredQuery!; @@ -840,7 +840,11 @@ export class JsonProtoSerializer { LimitType.First, startAt, endAt - ).toTarget(); + ); + } + + fromQueryTarget(target: api.QueryTarget): Target { + return this.convertQueryTargetToQuery(target).toTarget(); } toListenRequestLabels( diff --git a/packages/firestore/test/unit/local/bundle_cache.test.ts b/packages/firestore/test/unit/local/bundle_cache.test.ts new file mode 100644 index 00000000000..11c7b1c7e67 --- /dev/null +++ b/packages/firestore/test/unit/local/bundle_cache.test.ts @@ -0,0 +1,207 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; +import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; +import { filter, orderBy, path } from '../../util/helpers'; +import * as persistenceHelpers from './persistence_test_helpers'; +import { TestBundleCache } from './test_bundle_cache'; +import { SnapshotVersion } from '../../../src/core/snapshot_version'; +import { Timestamp } from '../../../src/api/timestamp'; +import { Query } from '../../../src/core/query'; +import { JSON_SERIALIZER } from './persistence_test_helpers'; +import { ResourcePath } from '../../../src/model/path'; +import { NamedQuery } from '../../../src/core/bundle'; + +describe('MemoryBundleCache', () => { + let cache: TestBundleCache; + + beforeEach(async () => { + cache = await persistenceHelpers + .testMemoryEagerPersistence() + .then(persistence => new TestBundleCache(persistence)); + }); + + genericBundleCacheTests(() => cache); +}); + +describe('IndexedDbBundleCache', () => { + if (!IndexedDbPersistence.isAvailable()) { + console.warn('No IndexedDB. Skipping IndexedDbBundleCache tests.'); + return; + } + + let cache: TestBundleCache; + let persistence: IndexedDbPersistence; + beforeEach(async () => { + persistence = await persistenceHelpers.testIndexedDbPersistence({ + synchronizeTabs: true + }); + cache = new TestBundleCache(persistence); + }); + + afterEach(async () => { + await persistence.shutdown(); + await persistenceHelpers.clearTestPersistence(); + }); + + genericBundleCacheTests(() => cache); +}); + +/** + * Defines the set of tests to run against both bundle cache implementations. + */ +function genericBundleCacheTests(cacheFn: () => TestBundleCache): void { + let cache: TestBundleCache; + + beforeEach(async () => { + cache = cacheFn(); + }); + + function verifyNamedQuery( + actual: NamedQuery, + expectedName: string, + expectedQuery: Query, + expectedReadSeconds: number, + expectedReadNanos: number + ):void { + expect(actual.name).to.equal(expectedName); + expect(actual.query.isEqual(expectedQuery)).to.be.true; + expect( + actual.readTime.isEqual( + SnapshotVersion.fromTimestamp( + new Timestamp(expectedReadSeconds, expectedReadNanos) + ) + ) + ).to.be.true; + } + + it('returns undefined when bundle id is not found', async () => { + expect(await cache.getBundle('bundle-1')).to.be.undefined; + }); + + it('returns saved bundle', async () => { + await cache.saveBundleMetadata({ + id: 'bundle-1', + version: 1, + createTime: { seconds: 1, nanos: 9999 } + }); + expect(await cache.getBundle('bundle-1')).to.deep.equal({ + id: 'bundle-1', + version: 1, + createTime: SnapshotVersion.fromTimestamp(new Timestamp(1, 9999)) + }); + + // Overwrite + await cache.saveBundleMetadata({ + id: 'bundle-1', + version: 2, + createTime: { seconds: 2, nanos: 1111 } + }); + expect(await cache.getBundle('bundle-1')).to.deep.equal({ + id: 'bundle-1', + version: 2, + createTime: SnapshotVersion.fromTimestamp(new Timestamp(2, 1111)) + }); + }); + + it('returns undefined when query name is not found', async () => { + expect(await cache.getNamedQuery('query-1')).to.be.undefined; + }); + + it('returns saved collection queries', async () => { + const query = Query.atPath(path('collection')) + .addFilter(filter('sort', '>=', 2)) + .addOrderBy(orderBy('sort')); + const queryTarget = JSON_SERIALIZER.toQueryTarget(query.toTarget()); + + await cache.setNamedQuery({ + name: 'query-1', + readTime: { seconds: 1, nanos: 9999 }, + bundledQuery: { + parent: queryTarget.parent, + structuredQuery: queryTarget.structuredQuery + } + }); + + const namedQuery = await cache.getNamedQuery('query-1'); + verifyNamedQuery(namedQuery!, 'query-1', query, 1, 9999); + }); + + it('returns saved collection group queries', async () => { + const query = new Query(ResourcePath.EMPTY_PATH, 'collection'); + const queryTarget = JSON_SERIALIZER.toQueryTarget(query.toTarget()); + + await cache.setNamedQuery({ + name: 'query-1', + readTime: { seconds: 1, nanos: 9999 }, + bundledQuery: { + parent: queryTarget.parent, + structuredQuery: queryTarget.structuredQuery, + limitType: undefined + } + }); + + const namedQuery = await cache.getNamedQuery('query-1'); + verifyNamedQuery(namedQuery!, 'query-1', query, 1, 9999); + }); + + it('returns expected limit queries', async () => { + const query = Query.atPath(path('collection')) + .addOrderBy(orderBy('sort')) + .withLimitToFirst(3); + const queryTarget = JSON_SERIALIZER.toQueryTarget(query.toTarget()); + + await cache.setNamedQuery({ + name: 'query-1', + readTime: { seconds: 1, nanos: 9999 }, + bundledQuery: { + parent: queryTarget.parent, + structuredQuery: queryTarget.structuredQuery, + limitType: 'FIRST' + } + }); + + const namedQuery = await cache.getNamedQuery('query-1'); + verifyNamedQuery(namedQuery!, 'query-1', query, 1, 9999); + }); + + it('returns expected limit to last queries', async () => { + const query = Query.atPath(path('collection')) + .addOrderBy(orderBy('sort')) + .withLimitToLast(3); + // Simulating bundle building for limit-to-last queries from the server + // SDKs: they save the equivelent limit-to-first queries with a limitType + // value 'LAST'. Client SDKs should apply a withLimitToLast when they see + // limitType 'LAST' from bundles. + const limitQuery = query.withLimitToFirst(3); + const queryTarget = JSON_SERIALIZER.toQueryTarget(limitQuery.toTarget()); + + await cache.setNamedQuery({ + name: 'query-1', + readTime: { seconds: 1, nanos: 9999 }, + bundledQuery: { + parent: queryTarget.parent, + structuredQuery: queryTarget.structuredQuery, + limitType: 'LAST' + } + }); + + const namedQuery = await cache.getNamedQuery('query-1'); + verifyNamedQuery(namedQuery!, 'query-1', query, 1, 9999); + }); +} diff --git a/packages/firestore/test/unit/local/persistence_test_helpers.ts b/packages/firestore/test/unit/local/persistence_test_helpers.ts index b60b32d4264..9733ae9e009 100644 --- a/packages/firestore/test/unit/local/persistence_test_helpers.ts +++ b/packages/firestore/test/unit/local/persistence_test_helpers.ts @@ -86,7 +86,7 @@ export const INDEXEDDB_TEST_DATABASE_NAME = IndexedDbPersistence.buildStoragePrefix(TEST_DATABASE_INFO) + IndexedDbPersistence.MAIN_DATABASE; -const JSON_SERIALIZER = new JsonProtoSerializer(TEST_DATABASE_ID, { +export const JSON_SERIALIZER = new JsonProtoSerializer(TEST_DATABASE_ID, { useProto3Json: true }); @@ -131,13 +131,16 @@ export async function testIndexedDbPersistence( /** Creates and starts a MemoryPersistence instance for testing. */ export async function testMemoryEagerPersistence(): Promise { - return new MemoryPersistence(MemoryEagerDelegate.factory); + return new MemoryPersistence(MemoryEagerDelegate.factory, JSON_SERIALIZER); } export async function testMemoryLruPersistence( params: LruParams = LruParams.DEFAULT ): Promise { - return new MemoryPersistence(p => new MemoryLruDelegate(p, params)); + return new MemoryPersistence( + p => new MemoryLruDelegate(p, params), + JSON_SERIALIZER + ); } /** Clears the persistence in tests */ diff --git a/packages/firestore/test/unit/local/test_bundle_cache.ts b/packages/firestore/test/unit/local/test_bundle_cache.ts new file mode 100644 index 00000000000..e0ac098952c --- /dev/null +++ b/packages/firestore/test/unit/local/test_bundle_cache.ts @@ -0,0 +1,73 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Persistence } from '../../../src/local/persistence'; +import { BundleCache } from '../../../src/local/bundle_cache'; +import { Bundle, NamedQuery } from '../../../src/core/bundle'; +import * as bundleProto from '../../../src/protos/firestore_bundle_proto'; + +/** + * A wrapper around a BundleCache that automatically creates a + * transaction around every operation to reduce test boilerplate. + */ +export class TestBundleCache { + private readonly cache: BundleCache; + + constructor(private readonly persistence: Persistence) { + this.cache = persistence.getBundleCache(); + } + + getBundle(bundleId: string): Promise { + return this.persistence.runTransaction( + 'getBundle', + 'readonly', + transaction => { + return this.cache.getBundle(transaction, bundleId); + } + ); + } + + saveBundleMetadata(metadata: bundleProto.BundleMetadata): Promise { + return this.persistence.runTransaction( + 'saveBundleMetadata', + 'readwrite', + transaction => { + return this.cache.saveBundleMetadata(transaction, metadata); + } + ); + } + + getNamedQuery(name: string): Promise { + return this.persistence.runTransaction( + 'getNamedQuery', + 'readonly', + transaction => { + return this.cache.getNamedQuery(transaction, name); + } + ); + } + + setNamedQuery(query: bundleProto.NamedQuery): Promise { + return this.persistence.runTransaction( + 'setNamedQuery', + 'readwrite', + transaction => { + return this.cache.saveNamedQuery(transaction, query); + } + ); + } +} diff --git a/packages/firestore/test/unit/specs/spec_test_components.ts b/packages/firestore/test/unit/specs/spec_test_components.ts index 9fba3e611a4..1da4d480d8c 100644 --- a/packages/firestore/test/unit/specs/spec_test_components.ts +++ b/packages/firestore/test/unit/specs/spec_test_components.ts @@ -51,6 +51,9 @@ import { ViewSnapshot } from '../../../src/core/view_snapshot'; import { Query } from '../../../src/core/query'; import { Mutation } from '../../../src/model/mutation'; import { expect } from 'chai'; +import { + JSON_SERIALIZER +} from '../local/persistence_test_helpers'; /** * A test-only MemoryPersistence implementation that is able to inject @@ -161,7 +164,8 @@ export class MockMemoryComponentProvider extends MemoryComponentProvider { return new MockMemoryPersistence( this.gcEnabled ? MemoryEagerDelegate.factory - : p => new MemoryLruDelegate(p, LruParams.DEFAULT) + : p => new MemoryLruDelegate(p, LruParams.DEFAULT), + JSON_SERIALIZER ); } } From fff3d36dad514d493f0cd91c583ad61e77b51d76 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Mon, 1 Jun 2020 15:35:02 -0400 Subject: [PATCH 13/39] Add applyBundleDocuments to local store. --- packages/firestore/src/core/bundle.ts | 37 ++++ .../firestore/src/core/component_provider.ts | 9 +- .../local/indexeddb_remote_document_cache.ts | 8 +- packages/firestore/src/local/local_store.ts | 191 ++++++++++++------ .../src/local/memory_remote_document_cache.ts | 6 +- .../local/remote_document_change_buffer.ts | 47 ++--- .../test/unit/local/local_store.test.ts | 146 ++++++++++++- .../remote_document_change_buffer.test.ts | 9 - packages/firestore/test/util/helpers.ts | 35 ++++ 9 files changed, 382 insertions(+), 106 deletions(-) diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index 0571664b7f4..52bea47d813 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -17,6 +17,16 @@ import { Query } from './query'; import { SnapshotVersion } from './snapshot_version'; +import { JsonProtoSerializer } from '../remote/serializer'; +import { + firestoreV1ApiClientInterfaces, + Timestamp +} from '../protos/firestore_proto_api'; +import Document = firestoreV1ApiClientInterfaces.Document; +import { DocumentKey } from '../model/document_key'; +import { MaybeDocument, NoDocument } from '../model/document'; +import { BundledDocumentMetadata } from '../protos/firestore_bundle_proto'; +import { debugAssert } from '../util/assert'; /** * Represents a Firestore bundle saved by the SDK in its local storage. @@ -37,3 +47,30 @@ export interface NamedQuery { // When the results for this query are read to the saved bundle. readonly readTime: SnapshotVersion; } + +export class BundleConverter { + constructor(private serializer: JsonProtoSerializer) {} + + toDocumentKey(name: string): DocumentKey { + return this.serializer.fromName(name); + } + + toMaybeDocument( + metadata: BundledDocumentMetadata, + doc: Document | undefined + ): MaybeDocument { + if (metadata.exists) { + debugAssert(!!doc, 'Document is undefined when metadata.exist is true.'); + return this.serializer.fromDocument(doc!, false); + } else { + return new NoDocument( + this.toDocumentKey(metadata.name!), + this.toSnapshotVersion(metadata.readTime!) + ); + } + } + + toSnapshotVersion(time: Timestamp): SnapshotVersion { + return this.serializer.fromVersion(time); + } +} diff --git a/packages/firestore/src/core/component_provider.ts b/packages/firestore/src/core/component_provider.ts index 38ba5d2c763..94ec74d71ed 100644 --- a/packages/firestore/src/core/component_provider.ts +++ b/packages/firestore/src/core/component_provider.ts @@ -42,6 +42,7 @@ import { MemoryEagerDelegate, MemoryPersistence } from '../local/memory_persistence'; +import { BundleConverter } from './bundle'; const MEMORY_ONLY_PERSISTENCE_ERROR_MESSAGE = 'You are using the memory-only build of Firestore. Persistence support is ' + @@ -125,10 +126,12 @@ export class MemoryComponentProvider implements ComponentProvider { } createLocalStore(cfg: ComponentConfiguration): LocalStore { + const serializer = cfg.platform.newSerializer(cfg.databaseInfo.databaseId); return new LocalStore( this.persistence, new IndexFreeQueryEngine(), - cfg.initialUser + cfg.initialUser, + new BundleConverter(serializer) ); } @@ -209,10 +212,12 @@ export class IndexedDbComponentProvider extends MemoryComponentProvider { } createLocalStore(cfg: ComponentConfiguration): LocalStore { + const serializer = cfg.platform.newSerializer(cfg.databaseInfo.databaseId); return new MultiTabLocalStore( this.persistence, new IndexFreeQueryEngine(), - cfg.initialUser + cfg.initialUser, + new BundleConverter(serializer) ); } diff --git a/packages/firestore/src/local/indexeddb_remote_document_cache.ts b/packages/firestore/src/local/indexeddb_remote_document_cache.ts index cf073f2b766..ad423ed82a5 100644 --- a/packages/firestore/src/local/indexeddb_remote_document_cache.ts +++ b/packages/firestore/src/local/indexeddb_remote_document_cache.ts @@ -453,12 +453,12 @@ export class IndexedDbRemoteDocumentCache implements RemoteDocumentCache { ); if (maybeDocument) { debugAssert( - !this.readTime.isEqual(SnapshotVersion.min()), + !this.getReadTime(key).isEqual(SnapshotVersion.min()), 'Cannot add a document with a read time of zero' ); const doc = this.documentCache.serializer.toDbRemoteDocument( - maybeDocument, - this.readTime + maybeDocument.maybeDoc!, + this.getReadTime(key) ); collectionParents = collectionParents.add(key.path.popLast()); @@ -474,7 +474,7 @@ export class IndexedDbRemoteDocumentCache implements RemoteDocumentCache { // preserved in `getNewDocumentChanges()`. const deletedDoc = this.documentCache.serializer.toDbRemoteDocument( new NoDocument(key, SnapshotVersion.min()), - this.readTime + this.getReadTime(key) ); promises.push( this.documentCache.addEntry(transaction, key, deletedDoc) diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 361cbd48b7f..c5cc857167b 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -25,6 +25,8 @@ import { DocumentKeySet, documentKeySet, DocumentMap, + documentVersionMap, + DocumentVersionMap, maybeDocumentMap, MaybeDocumentMap } from '../model/collections'; @@ -67,6 +69,9 @@ import { IndexedDbRemoteDocumentCache } from './indexeddb_remote_document_cache' import { IndexedDbTargetCache } from './indexeddb_target_cache'; import { extractFieldMask } from '../model/object_value'; import { isIndexedDbTransactionError } from './simple_db'; +import { BundledDocumentMetadata } from '../protos/firestore_bundle_proto'; +import * as api from '../protos/firestore_proto_api'; +import { BundleConverter } from '../core/bundle'; const LOG_TAG = 'LocalStore'; @@ -195,7 +200,8 @@ export class LocalStore { /** Manages our in-memory or durable persistence. */ protected persistence: Persistence, private queryEngine: QueryEngine, - initialUser: User + initialUser: User, + private bundleConverter: BundleConverter ) { debugAssert( persistence.started, @@ -485,10 +491,6 @@ export class LocalStore { return this.persistence .runTransaction('Apply remote event', 'readwrite-primary', txn => { - const documentBuffer = this.remoteDocuments.newChangeBuffer({ - trackRemovals: true // Make sure document removals show up in `getNewDocumentChanges()` - }); - // Reset newTargetDataByTargetMap in case this transaction gets re-run. newTargetDataByTargetMap = this.targetDataByTarget; @@ -541,65 +543,31 @@ export class LocalStore { } }); + const documentBuffer = this.remoteDocuments.newChangeBuffer({ + trackRemovals: true // Make sure document removals show up in `getNewDocumentChanges()` + }); let changedDocs = maybeDocumentMap(); let updatedKeys = documentKeySet(); remoteEvent.documentUpdates.forEach((key, doc) => { updatedKeys = updatedKeys.add(key); + if (remoteEvent.resolvedLimboDocuments.has(key)) { + promises.push( + this.persistence.referenceDelegate.updateLimboDocument(txn, key) + ); + } }); // Each loop iteration only affects its "own" doc, so it's safe to get all the remote // documents in advance in a single call. promises.push( - documentBuffer.getEntries(txn, updatedKeys).next(existingDocs => { - remoteEvent.documentUpdates.forEach((key, doc) => { - const existingDoc = existingDocs.get(key); - - // Note: The order of the steps below is important, since we want - // to ensure that rejected limbo resolutions (which fabricate - // NoDocuments with SnapshotVersion.min()) never add documents to - // cache. - if ( - doc instanceof NoDocument && - doc.version.isEqual(SnapshotVersion.min()) - ) { - // NoDocuments with SnapshotVersion.min() are used in manufactured - // events. We remove these documents from cache since we lost - // access. - documentBuffer.removeEntry(key, remoteVersion); - changedDocs = changedDocs.insert(key, doc); - } else if ( - existingDoc == null || - doc.version.compareTo(existingDoc.version) > 0 || - (doc.version.compareTo(existingDoc.version) === 0 && - existingDoc.hasPendingWrites) - ) { - debugAssert( - !SnapshotVersion.min().isEqual(remoteVersion), - 'Cannot add a document when the remote version is zero' - ); - documentBuffer.addEntry(doc, remoteVersion); - changedDocs = changedDocs.insert(key, doc); - } else { - logDebug( - LOG_TAG, - 'Ignoring outdated watch update for ', - key, - '. Current version:', - existingDoc.version, - ' Watch version:', - doc.version - ); - } - - if (remoteEvent.resolvedLimboDocuments.has(key)) { - promises.push( - this.persistence.referenceDelegate.updateLimboDocument( - txn, - key - ) - ); - } - }); + this.applyDocuments( + documentBuffer, + txn, + updatedKeys, + remoteEvent.documentUpdates, + remoteVersion + ).next(result => { + changedDocs = result; }) ); @@ -642,6 +610,114 @@ export class LocalStore { }); } + applyBundledDocuments( + documents: Array<[BundledDocumentMetadata, api.Document | undefined]> + ): Promise { + let updatedKeys = documentKeySet(); + let documentMap = maybeDocumentMap(); + let versionMap = documentVersionMap(); + for (const [metadata, doc] of documents) { + const documentKey = this.bundleConverter.toDocumentKey(metadata.name!); + updatedKeys = updatedKeys.add(documentKey); + documentMap = documentMap.insert( + documentKey, + this.bundleConverter.toMaybeDocument(metadata, doc) + ); + versionMap = versionMap.insert( + documentKey, + this.bundleConverter.toSnapshotVersion(metadata.readTime!) + ); + } + + const documentBuffer = this.remoteDocuments.newChangeBuffer({ + trackRemovals: true // Make sure document removals show up in `getNewDocumentChanges()` + }); + return this.persistence.runTransaction( + 'Apply bundle documents', + 'readwrite-primary', + txn => { + return this.applyDocuments( + documentBuffer, + txn, + updatedKeys, + documentMap, + versionMap + ) + .next(changedDocs => { + documentBuffer.apply(txn); + return changedDocs; + }) + .next(changedDocs => { + return this.localDocuments.getLocalViewOfDocuments( + txn, + changedDocs + ); + }); + } + ); + } + + private applyDocuments( + documentBuffer: RemoteDocumentChangeBuffer, + txn: PersistenceTransaction, + updatedKeys: DocumentKeySet, + documents: MaybeDocumentMap, + remoteVersion: SnapshotVersion | DocumentVersionMap + ): PersistencePromise { + const universalReadTime = remoteVersion instanceof SnapshotVersion; + return documentBuffer.getEntries(txn, updatedKeys).next(existingDocs => { + let changedDocs = maybeDocumentMap(); + documents.forEach((key, doc) => { + const existingDoc = existingDocs.get(key); + let docReadVersion: SnapshotVersion | null = null; + if (universalReadTime) { + docReadVersion = remoteVersion as SnapshotVersion; + } else { + docReadVersion = (remoteVersion as DocumentVersionMap).get(key); + } + debugAssert(!!docReadVersion, 'Document read version must exist'); + + // Note: The order of the steps below is important, since we want + // to ensure that rejected limbo resolutions (which fabricate + // NoDocuments with SnapshotVersion.min()) never add documents to + // cache. + if ( + doc instanceof NoDocument && + doc.version.isEqual(SnapshotVersion.min()) + ) { + // NoDocuments with SnapshotVersion.min() are used in manufactured + // events. We remove these documents from cache since we lost + // access. + documentBuffer.removeEntry(key, docReadVersion!); + changedDocs = changedDocs.insert(key, doc); + } else if ( + existingDoc == null || + doc.version.compareTo(existingDoc.version) > 0 || + (doc.version.compareTo(existingDoc.version) === 0 && + existingDoc.hasPendingWrites) + ) { + debugAssert( + !SnapshotVersion.min().isEqual(docReadVersion!), + 'Cannot add a document when the remote version is zero' + ); + documentBuffer.addEntry(doc, docReadVersion!); + changedDocs = changedDocs.insert(key, doc); + } else { + logDebug( + LOG_TAG, + 'Ignoring outdated watch update for ', + key, + '. Current version:', + existingDoc.version, + ' Watch version:', + doc.version + ); + } + }); + return changedDocs; + }); + } + /** * Returns true if the newTargetData should be persisted during an update of * an active target. TargetData should always be persisted when a target is @@ -1008,9 +1084,10 @@ export class MultiTabLocalStore extends LocalStore { constructor( protected persistence: IndexedDbPersistence, queryEngine: QueryEngine, - initialUser: User + initialUser: User, + bundleConverter: BundleConverter ) { - super(persistence, queryEngine, initialUser); + super(persistence, queryEngine, initialUser, bundleConverter); this.mutationQueue = persistence.getMutationQueue(initialUser); this.remoteDocuments = persistence.getRemoteDocumentCache(); diff --git a/packages/firestore/src/local/memory_remote_document_cache.ts b/packages/firestore/src/local/memory_remote_document_cache.ts index c8b147e6785..3d08abe2f9d 100644 --- a/packages/firestore/src/local/memory_remote_document_cache.ts +++ b/packages/firestore/src/local/memory_remote_document_cache.ts @@ -202,7 +202,11 @@ export class MemoryRemoteDocumentCache implements RemoteDocumentCache { this.changes.forEach((key, doc) => { if (doc) { promises.push( - this.documentCache.addEntry(transaction, doc, this.readTime) + this.documentCache.addEntry( + transaction, + doc.maybeDoc!, + doc.readTime! + ) ); } else { this.documentCache.removeEntry(key); diff --git a/packages/firestore/src/local/remote_document_change_buffer.ts b/packages/firestore/src/local/remote_document_change_buffer.ts index b434900afdb..c9fe583a144 100644 --- a/packages/firestore/src/local/remote_document_change_buffer.ts +++ b/packages/firestore/src/local/remote_document_change_buffer.ts @@ -25,6 +25,13 @@ import { PersistenceTransaction } from './persistence'; import { PersistencePromise } from './persistence_promise'; import { SnapshotVersion } from '../core/snapshot_version'; +class RemoteDocumentChange { + constructor( + readonly maybeDoc: MaybeDocument | null, + readonly readTime: SnapshotVersion + ) {} +} + /** * An in-memory buffer of entries to be written to a RemoteDocumentCache. * It can be used to batch up a set of changes to be written to the cache, but @@ -44,12 +51,9 @@ export abstract class RemoteDocumentChangeBuffer { // existing cache entry should be removed). protected changes: ObjectMap< DocumentKey, - MaybeDocument | null + RemoteDocumentChange | null > = new ObjectMap(key => key.toString()); - // The read time to use for all added documents in this change buffer. - private _readTime: SnapshotVersion | undefined; - private changesApplied = false; protected abstract getFromCache( @@ -66,23 +70,11 @@ export abstract class RemoteDocumentChangeBuffer { transaction: PersistenceTransaction ): PersistencePromise; - protected set readTime(value: SnapshotVersion) { - // Right now (for simplicity) we just track a single readTime for all the - // added entries since we expect them to all be the same, but we could - // rework to store per-entry readTimes if necessary. - debugAssert( - this._readTime === undefined || this._readTime.isEqual(value), - 'All changes in a RemoteDocumentChangeBuffer must have the same read time' - ); - this._readTime = value; - } - - protected get readTime(): SnapshotVersion { - debugAssert( - this._readTime !== undefined, - 'Read time is not set. All removeEntry() calls must include a readTime if `trackRemovals` is used.' - ); - return this._readTime; + protected getReadTime(key: DocumentKey): SnapshotVersion { + if (this.changes.get(key)) { + return this.changes.get(key)!.readTime; + } + return SnapshotVersion.min(); } /** @@ -93,8 +85,10 @@ export abstract class RemoteDocumentChangeBuffer { */ addEntry(maybeDocument: MaybeDocument, readTime: SnapshotVersion): void { this.assertNotApplied(); - this.readTime = readTime; - this.changes.set(maybeDocument.key, maybeDocument); + this.changes.set( + maybeDocument.key, + new RemoteDocumentChange(maybeDocument, readTime) + ); } /** @@ -105,9 +99,6 @@ export abstract class RemoteDocumentChangeBuffer { */ removeEntry(key: DocumentKey, readTime?: SnapshotVersion): void { this.assertNotApplied(); - if (readTime) { - this.readTime = readTime; - } this.changes.set(key, null); } @@ -129,7 +120,9 @@ export abstract class RemoteDocumentChangeBuffer { this.assertNotApplied(); const bufferedEntry = this.changes.get(documentKey); if (bufferedEntry !== undefined) { - return PersistencePromise.resolve(bufferedEntry); + return PersistencePromise.resolve( + (bufferedEntry || { maybeDoc: null }).maybeDoc + ); } else { return this.getFromCache(transaction, documentKey); } diff --git a/packages/firestore/test/unit/local/local_store.test.ts b/packages/firestore/test/unit/local/local_store.test.ts index c7a3ee3b6a6..fc2737a8edc 100644 --- a/packages/firestore/test/unit/local/local_store.test.ts +++ b/packages/firestore/test/unit/local/local_store.test.ts @@ -74,6 +74,8 @@ import { patchMutation, path, setMutation, + bundledDocuments, + TestBundledDocuments, TestSnapshotVersion, transformMutation, unknownDoc, @@ -83,6 +85,11 @@ import { import { CountingQueryEngine, QueryEngineType } from './counting_query_engine'; import * as persistenceHelpers from './persistence_test_helpers'; import { ByteString } from '../../../src/util/byte_string'; +import { BundleConverter } from '../../../src/core/bundle'; +import { JSON_SERIALIZER } from './persistence_test_helpers'; +import { BundledDocumentMetadata } from '../../../src/protos/firestore_bundle_proto'; +import { firestoreV1ApiClientInterfaces } from '../../../src/protos/firestore_proto_api'; +import Document = firestoreV1ApiClientInterfaces.Document; export interface LocalStoreComponents { queryEngine: CountingQueryEngine; @@ -111,7 +118,12 @@ class LocalStoreTester { } after( - op: Mutation | Mutation[] | RemoteEvent | LocalViewChanges + op: + | Mutation + | Mutation[] + | RemoteEvent + | LocalViewChanges + | TestBundledDocuments ): LocalStoreTester { if (op instanceof Mutation) { return this.afterMutations([op]); @@ -119,8 +131,10 @@ class LocalStoreTester { return this.afterMutations(op); } else if (op instanceof LocalViewChanges) { return this.afterViewChanges(op); - } else { + } else if (op instanceof RemoteEvent) { return this.afterRemoteEvent(op); + } else { + return this.afterBundleDocuments(op.documents); } } @@ -153,6 +167,21 @@ class LocalStoreTester { return this; } + afterBundleDocuments( + documents: Array<[BundledDocumentMetadata, Document | undefined]> + ): LocalStoreTester { + this.prepareNextStep(); + + this.promiseChain = this.promiseChain + .then(() => { + return this.localStore.applyBundledDocuments(documents); + }) + .then((result: MaybeDocumentMap) => { + this.lastChanges = result; + }); + return this; + } + afterViewChanges(viewChanges: LocalViewChanges): LocalStoreTester { this.prepareNextStep(); @@ -404,7 +433,8 @@ describe('LocalStore w/ Memory Persistence (SimpleQueryEngine)', () => { const localStore = new LocalStore( persistence, queryEngine, - User.UNAUTHENTICATED + User.UNAUTHENTICATED, + new BundleConverter(JSON_SERIALIZER) ); return { queryEngine, persistence, localStore }; } @@ -422,7 +452,8 @@ describe('LocalStore w/ Memory Persistence (IndexFreeQueryEngine)', () => { const localStore = new LocalStore( persistence, queryEngine, - User.UNAUTHENTICATED + User.UNAUTHENTICATED, + new BundleConverter(JSON_SERIALIZER) ); return { queryEngine, persistence, localStore }; } @@ -448,7 +479,8 @@ describe('LocalStore w/ IndexedDB Persistence (SimpleQueryEngine)', () => { const localStore = new MultiTabLocalStore( persistence, queryEngine, - User.UNAUTHENTICATED + User.UNAUTHENTICATED, + new BundleConverter(JSON_SERIALIZER) ); await localStore.start(); return { queryEngine, persistence, localStore }; @@ -475,7 +507,8 @@ describe('LocalStore w/ IndexedDB Persistence (IndexFreeQueryEngine)', () => { const localStore = new MultiTabLocalStore( persistence, queryEngine, - User.UNAUTHENTICATED + User.UNAUTHENTICATED, + new BundleConverter(JSON_SERIALIZER) ); await localStore.start(); return { queryEngine, persistence, localStore }; @@ -1486,6 +1519,107 @@ function genericLocalStoreTests( .finish(); }); + it('handles saving bundled documents', () => { + return expectLocalStore() + .after( + bundledDocuments([ + doc('foo/bar', 1, { sum: 1337 }), + deletedDoc('foo/bar1', 1) + ]) + ) + .toReturnChanged( + doc('foo/bar', 1, { sum: 1337 }), + deletedDoc('foo/bar1', 1) + ) + .toContain(doc('foo/bar', 1, { sum: 1337 })) + .toContain(deletedDoc('foo/bar1', 1)) + .finish(); + }); + + it('handles saving bundled documents with newer existing version', () => { + const query = Query.atPath(path('foo')); + return expectLocalStore() + .afterAllocatingQuery(query) + .toReturnTargetId(2) + .after(docAddedRemoteEvent(doc('foo/bar', 2, { sum: 1337 }), [2])) + .toContain(doc('foo/bar', 2, { sum: 1337 })) + .after( + bundledDocuments([ + doc('foo/bar', 1, { sum: 1336 }), + deletedDoc('foo/bar1', 1) + ]) + ) + .toReturnChanged(deletedDoc('foo/bar1', 1)) + .toContain(doc('foo/bar', 2, { sum: 1337 })) + .toContain(deletedDoc('foo/bar1', 1)) + .finish(); + }); + + it('handles saving bundled documents with older existing version', () => { + const query = Query.atPath(path('foo')); + return expectLocalStore() + .afterAllocatingQuery(query) + .toReturnTargetId(2) + .after(docAddedRemoteEvent(doc('foo/bar1', 1, { val: 'to-delete' }), [2])) + .toContain(doc('foo/bar1', 1, { val: 'to-delete' })) + .after( + bundledDocuments([ + doc('foo/bar', 1, { sum: 1336 }), + deletedDoc('foo/bar1', 2) + ]) + ) + .toReturnChanged( + doc('foo/bar', 1, { sum: 1336 }), + deletedDoc('foo/bar1', 2) + ) + .toContain(doc('foo/bar', 1, { sum: 1336 })) + .toContain(deletedDoc('foo/bar1', 2)) + .finish(); + }); + + it('handles MergeMutation with Transform -> BundledDocuments', () => { + const query = Query.atPath(path('foo')); + return expectLocalStore() + .afterAllocatingQuery(query) + .toReturnTargetId(2) + .afterMutations([ + patchMutation('foo/bar', {}, Precondition.none()), + transformMutation('foo/bar', { sum: FieldValue.increment(1) }) + ]) + .toReturnChanged( + doc('foo/bar', 0, { sum: 1 }, { hasLocalMutations: true }) + ) + .toContain(doc('foo/bar', 0, { sum: 1 }, { hasLocalMutations: true })) + .after(bundledDocuments([doc('foo/bar', 1, { sum: 1337 })])) + .toReturnChanged( + doc('foo/bar', 1, { sum: 1 }, { hasLocalMutations: true }) + ) + .toContain(doc('foo/bar', 1, { sum: 1 }, { hasLocalMutations: true })) + .finish(); + }); + + it('handles PatchMutation with Transform -> BundledDocuments', () => { + // Note: This test reflects the current behavior, but it may be preferable + // to replay the mutation once we receive the first value from the backend. + + const query = Query.atPath(path('foo')); + return expectLocalStore() + .afterAllocatingQuery(query) + .toReturnTargetId(2) + .afterMutations([ + patchMutation('foo/bar', {}), + transformMutation('foo/bar', { sum: FieldValue.increment(1) }) + ]) + .toReturnChanged(deletedDoc('foo/bar', 0)) + .toNotContain('foo/bar') + .after(bundledDocuments([doc('foo/bar', 1, { sum: 1337 })])) + .toReturnChanged( + doc('foo/bar', 1, { sum: 1 }, { hasLocalMutations: true }) + ) + .toContain(doc('foo/bar', 1, { sum: 1 }, { hasLocalMutations: true })) + .finish(); + }); + it('computes highest unacknowledged batch id correctly', () => { return expectLocalStore() .toReturnHighestUnacknowledgeBatchId(BATCHID_UNKNOWN) diff --git a/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts b/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts index ec77f35695d..043e5905434 100644 --- a/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts +++ b/packages/firestore/test/unit/local/remote_document_change_buffer.test.ts @@ -116,13 +116,4 @@ describe('RemoteDocumentChangeBuffer', () => { .catch(() => errors++) .then(() => expect(errors).to.equal(2)); }); - - it('cannot add documents with different read times', async () => { - // This test merely validates an assert that was added to the - // RemoteDocumentChangeBuffer to simplify our tracking of document - // read times. If we do need to track documents with different read - // times, this test should simply be removed. - buffer.addEntry(INITIAL_DOC, version(1)); - expect(() => buffer.addEntry(INITIAL_DOC, version(2))).to.throw(); - }); }); diff --git a/packages/firestore/test/util/helpers.ts b/packages/firestore/test/util/helpers.ts index 8dc7545daf2..02c7f6ab3ba 100644 --- a/packages/firestore/test/util/helpers.ts +++ b/packages/firestore/test/util/helpers.ts @@ -92,6 +92,8 @@ import { Timestamp } from '../../src/api/timestamp'; import { DocumentReference, Firestore } from '../../src/api/database'; import { DeleteFieldValueImpl } from '../../src/api/field_value'; import { Code, FirestoreError } from '../../src/util/error'; +import { BundledDocumentMetadata } from '../../src/protos/firestore_bundle_proto'; +import { JSON_SERIALIZER } from '../unit/local/persistence_test_helpers'; /* eslint-disable no-restricted-globals */ @@ -401,6 +403,39 @@ export function docUpdateRemoteEvent( return aggregator.createRemoteEvent(doc.version); } +export class TestBundledDocuments { + constructor( + public documents: Array<[BundledDocumentMetadata, api.Document | undefined]> + ) {} +} + +export function bundledDocuments(documents: MaybeDocument[]) { + let result: Array<[BundledDocumentMetadata, api.Document | undefined]> = []; + for (const d of documents) { + if (d instanceof NoDocument) { + result.push([ + { + name: JSON_SERIALIZER.toName(d.key), + readTime: JSON_SERIALIZER.toVersion(d.version), + exists: false + }, + undefined + ]); + } else if (d instanceof Document) { + result.push([ + { + name: JSON_SERIALIZER.toName(d.key), + readTime: JSON_SERIALIZER.toVersion(d.version), + exists: true + }, + JSON_SERIALIZER.toDocument(d) + ]); + } + } + + return new TestBundledDocuments(result); +} + export function updateMapping( snapshotVersion: SnapshotVersion, added: Array, From fb762de46cdc6b2af65a81f168594ddefc517ddd Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Mon, 1 Jun 2020 16:50:41 -0400 Subject: [PATCH 14/39] Add rest of bundle service to localstore Fix change buffer bug --- packages/firestore/src/core/bundle.ts | 18 +++--- .../local/indexeddb_remote_document_cache.ts | 6 +- packages/firestore/src/local/local_store.ts | 56 ++++++++++++++++++- .../src/local/memory_remote_document_cache.ts | 4 +- .../local/remote_document_change_buffer.ts | 10 ++-- .../test/unit/local/local_store.test.ts | 6 +- packages/firestore/test/util/helpers.ts | 11 ++-- 7 files changed, 80 insertions(+), 31 deletions(-) diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index 52bea47d813..bcd8bedcfe0 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -18,14 +18,10 @@ import { Query } from './query'; import { SnapshotVersion } from './snapshot_version'; import { JsonProtoSerializer } from '../remote/serializer'; -import { - firestoreV1ApiClientInterfaces, - Timestamp -} from '../protos/firestore_proto_api'; -import Document = firestoreV1ApiClientInterfaces.Document; +import * as bundleProto from '../protos/firestore_bundle_proto'; +import * as api from '../protos/firestore_proto_api'; import { DocumentKey } from '../model/document_key'; import { MaybeDocument, NoDocument } from '../model/document'; -import { BundledDocumentMetadata } from '../protos/firestore_bundle_proto'; import { debugAssert } from '../util/assert'; /** @@ -48,6 +44,10 @@ export interface NamedQuery { readonly readTime: SnapshotVersion; } +export type BundledDocuments = Array< + [bundleProto.BundledDocumentMetadata, api.Document | undefined] +>; + export class BundleConverter { constructor(private serializer: JsonProtoSerializer) {} @@ -56,8 +56,8 @@ export class BundleConverter { } toMaybeDocument( - metadata: BundledDocumentMetadata, - doc: Document | undefined + metadata: bundleProto.BundledDocumentMetadata, + doc: api.Document | undefined ): MaybeDocument { if (metadata.exists) { debugAssert(!!doc, 'Document is undefined when metadata.exist is true.'); @@ -70,7 +70,7 @@ export class BundleConverter { } } - toSnapshotVersion(time: Timestamp): SnapshotVersion { + toSnapshotVersion(time: api.Timestamp): SnapshotVersion { return this.serializer.fromVersion(time); } } diff --git a/packages/firestore/src/local/indexeddb_remote_document_cache.ts b/packages/firestore/src/local/indexeddb_remote_document_cache.ts index ad423ed82a5..8b78428fae0 100644 --- a/packages/firestore/src/local/indexeddb_remote_document_cache.ts +++ b/packages/firestore/src/local/indexeddb_remote_document_cache.ts @@ -445,19 +445,19 @@ export class IndexedDbRemoteDocumentCache implements RemoteDocumentCache { primitiveComparator(l.canonicalString(), r.canonicalString()) ); - this.changes.forEach((key, maybeDocument) => { + this.changes.forEach((key, documentChange) => { const previousSize = this.documentSizes.get(key); debugAssert( previousSize !== undefined, `Cannot modify a document that wasn't read (for ${key})` ); - if (maybeDocument) { + if (documentChange.maybeDoc) { debugAssert( !this.getReadTime(key).isEqual(SnapshotVersion.min()), 'Cannot add a document with a read time of zero' ); const doc = this.documentCache.serializer.toDbRemoteDocument( - maybeDocument.maybeDoc!, + documentChange.maybeDoc!, this.getReadTime(key) ); collectionParents = collectionParents.add(key.path.popLast()); diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index c5cc857167b..4d53f9171c8 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -69,9 +69,15 @@ import { IndexedDbRemoteDocumentCache } from './indexeddb_remote_document_cache' import { IndexedDbTargetCache } from './indexeddb_target_cache'; import { extractFieldMask } from '../model/object_value'; import { isIndexedDbTransactionError } from './simple_db'; -import { BundledDocumentMetadata } from '../protos/firestore_bundle_proto'; +import * as bundleProto from '../protos/firestore_bundle_proto'; import * as api from '../protos/firestore_proto_api'; -import { BundleConverter } from '../core/bundle'; +import { + Bundle, + BundleConverter, + BundledDocuments, + NamedQuery +} from '../core/bundle'; +import { BundleCache } from './bundle_cache'; const LOG_TAG = 'LocalStore'; @@ -173,6 +179,9 @@ export class LocalStore { /** Maps a target to its `TargetData`. */ protected targetCache: TargetCache; + /** The set of all cached bundle metadata and named queries. */ + protected bundleCache: BundleCache; + /** * Maps a targetID to data about its target. * @@ -210,6 +219,7 @@ export class LocalStore { this.mutationQueue = persistence.getMutationQueue(initialUser); this.remoteDocuments = persistence.getRemoteDocumentCache(); this.targetCache = persistence.getTargetCache(); + this.bundleCache = persistence.getBundleCache(); this.localDocuments = new LocalDocumentsView( this.remoteDocuments, this.mutationQueue, @@ -611,7 +621,7 @@ export class LocalStore { } applyBundledDocuments( - documents: Array<[BundledDocumentMetadata, api.Document | undefined]> + documents: BundledDocuments ): Promise { let updatedKeys = documentKeySet(); let documentMap = maybeDocumentMap(); @@ -718,6 +728,46 @@ export class LocalStore { }); } + getBundle(bundleId: string): Promise { + return this.persistence.runTransaction( + 'Get bundle', + 'readonly', + transaction => { + return this.bundleCache.getBundle(transaction, bundleId); + } + ); + } + + saveBundle(bundleMetadata: bundleProto.BundleMetadata): Promise { + return this.persistence.runTransaction( + 'Save bundle', + 'readwrite', + transaction => { + return this.bundleCache.saveBundleMetadata(transaction, bundleMetadata); + } + ); + } + + getNamedQuery(queryName: string): Promise { + return this.persistence.runTransaction( + 'Get named query', + 'readonly', + transaction => { + return this.bundleCache.getNamedQuery(transaction, queryName); + } + ); + } + + saveNamedQuery(query: bundleProto.NamedQuery): Promise { + return this.persistence.runTransaction( + 'Save named query', + 'readwrite', + transaction => { + return this.bundleCache.saveNamedQuery(transaction, query); + } + ); + } + /** * Returns true if the newTargetData should be persisted during an update of * an active target. TargetData should always be persisted when a target is diff --git a/packages/firestore/src/local/memory_remote_document_cache.ts b/packages/firestore/src/local/memory_remote_document_cache.ts index 3d08abe2f9d..ced4b868774 100644 --- a/packages/firestore/src/local/memory_remote_document_cache.ts +++ b/packages/firestore/src/local/memory_remote_document_cache.ts @@ -200,12 +200,12 @@ export class MemoryRemoteDocumentCache implements RemoteDocumentCache { ): PersistencePromise { const promises: Array> = []; this.changes.forEach((key, doc) => { - if (doc) { + if (doc && doc.maybeDoc) { promises.push( this.documentCache.addEntry( transaction, doc.maybeDoc!, - doc.readTime! + this.getReadTime(key) ) ); } else { diff --git a/packages/firestore/src/local/remote_document_change_buffer.ts b/packages/firestore/src/local/remote_document_change_buffer.ts index c9fe583a144..9aabb4af2b7 100644 --- a/packages/firestore/src/local/remote_document_change_buffer.ts +++ b/packages/firestore/src/local/remote_document_change_buffer.ts @@ -28,7 +28,7 @@ import { SnapshotVersion } from '../core/snapshot_version'; class RemoteDocumentChange { constructor( readonly maybeDoc: MaybeDocument | null, - readonly readTime: SnapshotVersion + readonly readTime?: SnapshotVersion ) {} } @@ -51,7 +51,7 @@ export abstract class RemoteDocumentChangeBuffer { // existing cache entry should be removed). protected changes: ObjectMap< DocumentKey, - RemoteDocumentChange | null + RemoteDocumentChange > = new ObjectMap(key => key.toString()); private changesApplied = false; @@ -71,8 +71,8 @@ export abstract class RemoteDocumentChangeBuffer { ): PersistencePromise; protected getReadTime(key: DocumentKey): SnapshotVersion { - if (this.changes.get(key)) { - return this.changes.get(key)!.readTime; + if (this.changes.get(key) && this.changes.get(key)!.readTime) { + return this.changes.get(key)!.readTime!; } return SnapshotVersion.min(); } @@ -99,7 +99,7 @@ export abstract class RemoteDocumentChangeBuffer { */ removeEntry(key: DocumentKey, readTime?: SnapshotVersion): void { this.assertNotApplied(); - this.changes.set(key, null); + this.changes.set(key, new RemoteDocumentChange(null, readTime)); } /** diff --git a/packages/firestore/test/unit/local/local_store.test.ts b/packages/firestore/test/unit/local/local_store.test.ts index fc2737a8edc..1967262a236 100644 --- a/packages/firestore/test/unit/local/local_store.test.ts +++ b/packages/firestore/test/unit/local/local_store.test.ts @@ -85,7 +85,7 @@ import { import { CountingQueryEngine, QueryEngineType } from './counting_query_engine'; import * as persistenceHelpers from './persistence_test_helpers'; import { ByteString } from '../../../src/util/byte_string'; -import { BundleConverter } from '../../../src/core/bundle'; +import { BundleConverter, BundledDocuments } from '../../../src/core/bundle'; import { JSON_SERIALIZER } from './persistence_test_helpers'; import { BundledDocumentMetadata } from '../../../src/protos/firestore_bundle_proto'; import { firestoreV1ApiClientInterfaces } from '../../../src/protos/firestore_proto_api'; @@ -167,9 +167,7 @@ class LocalStoreTester { return this; } - afterBundleDocuments( - documents: Array<[BundledDocumentMetadata, Document | undefined]> - ): LocalStoreTester { + afterBundleDocuments(documents: BundledDocuments): LocalStoreTester { this.prepareNextStep(); this.promiseChain = this.promiseChain diff --git a/packages/firestore/test/util/helpers.ts b/packages/firestore/test/util/helpers.ts index 02c7f6ab3ba..d903944347d 100644 --- a/packages/firestore/test/util/helpers.ts +++ b/packages/firestore/test/util/helpers.ts @@ -94,6 +94,7 @@ import { DeleteFieldValueImpl } from '../../src/api/field_value'; import { Code, FirestoreError } from '../../src/util/error'; import { BundledDocumentMetadata } from '../../src/protos/firestore_bundle_proto'; import { JSON_SERIALIZER } from '../unit/local/persistence_test_helpers'; +import { BundledDocuments } from '../../src/core/bundle'; /* eslint-disable no-restricted-globals */ @@ -404,13 +405,13 @@ export function docUpdateRemoteEvent( } export class TestBundledDocuments { - constructor( - public documents: Array<[BundledDocumentMetadata, api.Document | undefined]> - ) {} + constructor(public documents: BundledDocuments) {} } -export function bundledDocuments(documents: MaybeDocument[]) { - let result: Array<[BundledDocumentMetadata, api.Document | undefined]> = []; +export function bundledDocuments( + documents: MaybeDocument[] +): TestBundledDocuments { + let result: BundledDocuments = []; for (const d of documents) { if (d instanceof NoDocument) { result.push([ From 1ec4182732ea261c447addc4ae18d8a94a882954 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 2 Jun 2020 11:30:08 -0400 Subject: [PATCH 15/39] Simplify change buffer get read time logic. --- .../src/local/remote_document_change_buffer.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/firestore/src/local/remote_document_change_buffer.ts b/packages/firestore/src/local/remote_document_change_buffer.ts index 9aabb4af2b7..747cd35bff6 100644 --- a/packages/firestore/src/local/remote_document_change_buffer.ts +++ b/packages/firestore/src/local/remote_document_change_buffer.ts @@ -71,8 +71,13 @@ export abstract class RemoteDocumentChangeBuffer { ): PersistencePromise; protected getReadTime(key: DocumentKey): SnapshotVersion { - if (this.changes.get(key) && this.changes.get(key)!.readTime) { - return this.changes.get(key)!.readTime!; + const change = this.changes.get(key); + if (change) { + debugAssert( + !!change.readTime, + `Read time is not set for ${key}. All removeEntry() calls must include a readTime if 'trackRemovals' is used.` + ); + return change.readTime!; } return SnapshotVersion.min(); } @@ -121,7 +126,7 @@ export abstract class RemoteDocumentChangeBuffer { const bufferedEntry = this.changes.get(documentKey); if (bufferedEntry !== undefined) { return PersistencePromise.resolve( - (bufferedEntry || { maybeDoc: null }).maybeDoc + bufferedEntry.maybeDoc ); } else { return this.getFromCache(transaction, documentKey); From cd3ab7abee88c45ed27555f5175d3ca61176bf0a Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 2 Jun 2020 11:51:23 -0400 Subject: [PATCH 16/39] Fix lint errors --- packages/firestore/src/local/local_store.ts | 1 - packages/firestore/test/unit/local/local_store.test.ts | 3 --- packages/firestore/test/util/helpers.ts | 3 +-- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 4d53f9171c8..69bef566fec 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -70,7 +70,6 @@ import { IndexedDbTargetCache } from './indexeddb_target_cache'; import { extractFieldMask } from '../model/object_value'; import { isIndexedDbTransactionError } from './simple_db'; import * as bundleProto from '../protos/firestore_bundle_proto'; -import * as api from '../protos/firestore_proto_api'; import { Bundle, BundleConverter, diff --git a/packages/firestore/test/unit/local/local_store.test.ts b/packages/firestore/test/unit/local/local_store.test.ts index 1967262a236..6b197525897 100644 --- a/packages/firestore/test/unit/local/local_store.test.ts +++ b/packages/firestore/test/unit/local/local_store.test.ts @@ -87,9 +87,6 @@ import * as persistenceHelpers from './persistence_test_helpers'; import { ByteString } from '../../../src/util/byte_string'; import { BundleConverter, BundledDocuments } from '../../../src/core/bundle'; import { JSON_SERIALIZER } from './persistence_test_helpers'; -import { BundledDocumentMetadata } from '../../../src/protos/firestore_bundle_proto'; -import { firestoreV1ApiClientInterfaces } from '../../../src/protos/firestore_proto_api'; -import Document = firestoreV1ApiClientInterfaces.Document; export interface LocalStoreComponents { queryEngine: CountingQueryEngine; diff --git a/packages/firestore/test/util/helpers.ts b/packages/firestore/test/util/helpers.ts index d903944347d..58d0620e647 100644 --- a/packages/firestore/test/util/helpers.ts +++ b/packages/firestore/test/util/helpers.ts @@ -92,7 +92,6 @@ import { Timestamp } from '../../src/api/timestamp'; import { DocumentReference, Firestore } from '../../src/api/database'; import { DeleteFieldValueImpl } from '../../src/api/field_value'; import { Code, FirestoreError } from '../../src/util/error'; -import { BundledDocumentMetadata } from '../../src/protos/firestore_bundle_proto'; import { JSON_SERIALIZER } from '../unit/local/persistence_test_helpers'; import { BundledDocuments } from '../../src/core/bundle'; @@ -411,7 +410,7 @@ export class TestBundledDocuments { export function bundledDocuments( documents: MaybeDocument[] ): TestBundledDocuments { - let result: BundledDocuments = []; + const result: BundledDocuments = []; for (const d of documents) { if (d instanceof NoDocument) { result.push([ From d991c7545e9b79d39cfbde7b8e314acddadfceae Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 2 Jun 2020 13:41:31 -0400 Subject: [PATCH 17/39] Add comments. --- packages/firestore/src/local/local_store.ts | 39 +++++++++++++++++-- .../local/remote_document_change_buffer.ts | 5 +++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 69bef566fec..4d843e03bf3 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -487,9 +487,10 @@ export class LocalStore { } /** - * Update the "ground-state" (remote) documents. We assume that the remote - * event reflects any write batches that have been acknowledged or rejected - * (i.e. we do not re-apply local mutations to updates from this event). + * Applies the documents from a remote event to the "ground-state" (remote) + * documents. We assume that the remote event reflects any write batches that + * have been acknowledged or rejected (i.e. we do not re-apply local + * mutations to updates from this event). * * LocalDocuments are re-calculated if there are remaining mutations in the * queue. @@ -619,6 +620,13 @@ export class LocalStore { }); } + /** + * Applies the documents from a bundle to the "ground-state" (remote) + * documents. + * + * LocalDocuments are re-calculated if there are remaining mutations in the + * queue. + */ applyBundledDocuments( documents: BundledDocuments ): Promise { @@ -666,6 +674,16 @@ export class LocalStore { ); } + /** + * Applies documents to remote document cache, returns the document changes + * resulting from applying those documents. + * + * @param updatedKeys Keys of the documents to be applied. + * @param documents Documents to be applied. + * @param remoteVersion The read time of the documents to be applied, it is + * a `SnapshotVersion` if all documents have the same read time, or a `DocumentVersionMap` if + * they have different read times. + */ private applyDocuments( documentBuffer: RemoteDocumentChangeBuffer, txn: PersistenceTransaction, @@ -727,6 +745,10 @@ export class LocalStore { }); } + /** + * Returns a promise of a `Bundle` associated with given bundle id. Promise + * resolves to undefined if no persisted data can be found. + */ getBundle(bundleId: string): Promise { return this.persistence.runTransaction( 'Get bundle', @@ -737,6 +759,10 @@ export class LocalStore { ); } + /** + * Saves the given `BundleMetadata` to local persistence. + * @param bundleMetadata + */ saveBundle(bundleMetadata: bundleProto.BundleMetadata): Promise { return this.persistence.runTransaction( 'Save bundle', @@ -747,6 +773,10 @@ export class LocalStore { ); } + /** + * Returns a promise of a `NamedQuery` associated with given query name. Promise + * resolves to undefined if no persisted data can be found. + */ getNamedQuery(queryName: string): Promise { return this.persistence.runTransaction( 'Get named query', @@ -757,6 +787,9 @@ export class LocalStore { ); } + /** + * Saves the given `NamedQuery` to local persistence. + */ saveNamedQuery(query: bundleProto.NamedQuery): Promise { return this.persistence.runTransaction( 'Save named query', diff --git a/packages/firestore/src/local/remote_document_change_buffer.ts b/packages/firestore/src/local/remote_document_change_buffer.ts index 747cd35bff6..6c0e050ee65 100644 --- a/packages/firestore/src/local/remote_document_change_buffer.ts +++ b/packages/firestore/src/local/remote_document_change_buffer.ts @@ -25,9 +25,14 @@ import { PersistenceTransaction } from './persistence'; import { PersistencePromise } from './persistence_promise'; import { SnapshotVersion } from '../core/snapshot_version'; +/** + * Represents a document change to be applied to remote document cache. + */ class RemoteDocumentChange { constructor( + // The document in this change, null if it is a removal from the cache. readonly maybeDoc: MaybeDocument | null, + // The timestamp when this change is read. readonly readTime?: SnapshotVersion ) {} } From af097c55613e3b5a3432e3f01815107e0dd76e8d Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 2 Jun 2020 17:27:01 -0400 Subject: [PATCH 18/39] Change localstore to check for newer bundle directly. --- packages/firestore/src/local/local_store.ts | 30 +++++++++--------- .../test/unit/local/local_store.test.ts | 31 ++++++++++++++++++- packages/firestore/test/util/helpers.ts | 17 ++++++++++ 3 files changed, 62 insertions(+), 16 deletions(-) diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 4d843e03bf3..c2005122726 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -70,12 +70,7 @@ import { IndexedDbTargetCache } from './indexeddb_target_cache'; import { extractFieldMask } from '../model/object_value'; import { isIndexedDbTransactionError } from './simple_db'; import * as bundleProto from '../protos/firestore_bundle_proto'; -import { - Bundle, - BundleConverter, - BundledDocuments, - NamedQuery -} from '../core/bundle'; +import { BundleConverter, BundledDocuments, NamedQuery } from '../core/bundle'; import { BundleCache } from './bundle_cache'; const LOG_TAG = 'LocalStore'; @@ -746,17 +741,22 @@ export class LocalStore { } /** - * Returns a promise of a `Bundle` associated with given bundle id. Promise - * resolves to undefined if no persisted data can be found. + * Returns a promise of a boolean to indicate if the given bundle has already + * been loaded and the create time is newer than the current loading bundle. */ - getBundle(bundleId: string): Promise { - return this.persistence.runTransaction( - 'Get bundle', - 'readonly', - transaction => { - return this.bundleCache.getBundle(transaction, bundleId); - } + isNewerBundleLoaded( + bundleMetadata: bundleProto.BundleMetadata + ): Promise { + const currentReadTime = this.bundleConverter.toSnapshotVersion( + bundleMetadata.createTime! ); + return this.persistence + .runTransaction('isNewerBundleLoaded', 'readonly', transaction => { + return this.bundleCache.getBundle(transaction, bundleMetadata.id!); + }) + .then(cached => { + return !!cached && cached!.createTime!.compareTo(currentReadTime) > 0; + }); } /** diff --git a/packages/firestore/test/unit/local/local_store.test.ts b/packages/firestore/test/unit/local/local_store.test.ts index 6b197525897..99d8b49d299 100644 --- a/packages/firestore/test/unit/local/local_store.test.ts +++ b/packages/firestore/test/unit/local/local_store.test.ts @@ -79,7 +79,8 @@ import { TestSnapshotVersion, transformMutation, unknownDoc, - version + version, + bundleMetadata } from '../../util/helpers'; import { CountingQueryEngine, QueryEngineType } from './counting_query_engine'; @@ -87,6 +88,7 @@ import * as persistenceHelpers from './persistence_test_helpers'; import { ByteString } from '../../../src/util/byte_string'; import { BundleConverter, BundledDocuments } from '../../../src/core/bundle'; import { JSON_SERIALIZER } from './persistence_test_helpers'; +import { BundleMetadata } from '../../../src/protos/firestore_bundle_proto'; export interface LocalStoreComponents { queryEngine: CountingQueryEngine; @@ -411,6 +413,25 @@ class LocalStoreTester { return this; } + toHaveNewerBundle( + metadata: BundleMetadata, + expected: boolean + ): LocalStoreTester { + this.promiseChain = this.promiseChain.then(() => { + return this.localStore.isNewerBundleLoaded(metadata).then(actual => { + expect(actual).to.equal(expected); + }); + }); + return this; + } + + afterSavingBundle(metadata: BundleMetadata): LocalStoreTester { + this.promiseChain = this.promiseChain.then(() => { + return this.localStore.saveBundle(metadata); + }); + return this; + } + finish(): Promise { return this.promiseChain; } @@ -1615,6 +1636,14 @@ function genericLocalStoreTests( .finish(); }); + it('handles saving and checking bundle metadata', () => { + return expectLocalStore() + .toHaveNewerBundle(bundleMetadata('test', 2), false) + .afterSavingBundle(bundleMetadata('test', 2)) + .toHaveNewerBundle(bundleMetadata('test', 1), true) + .finish(); + }); + it('computes highest unacknowledged batch id correctly', () => { return expectLocalStore() .toReturnHighestUnacknowledgeBatchId(BATCHID_UNKNOWN) diff --git a/packages/firestore/test/util/helpers.ts b/packages/firestore/test/util/helpers.ts index 58d0620e647..a92a1e2505f 100644 --- a/packages/firestore/test/util/helpers.ts +++ b/packages/firestore/test/util/helpers.ts @@ -94,6 +94,7 @@ import { DeleteFieldValueImpl } from '../../src/api/field_value'; import { Code, FirestoreError } from '../../src/util/error'; import { JSON_SERIALIZER } from '../unit/local/persistence_test_helpers'; import { BundledDocuments } from '../../src/core/bundle'; +import { BundleMetadata } from '../../src/protos/firestore_bundle_proto'; /* eslint-disable no-restricted-globals */ @@ -436,6 +437,22 @@ export function bundledDocuments( return new TestBundledDocuments(result); } +export function bundleMetadata( + id: string, + createTime: TestSnapshotVersion, + version = 1, + totalDocuments = 1, + totalBytes = 1000 +): BundleMetadata { + return { + id, + createTime: { seconds: createTime, nanos: 0 }, + version, + totalDocuments, + totalBytes + }; +} + export function updateMapping( snapshotVersion: SnapshotVersion, added: Array, From e735e238e01e7e587835309c0ac8dac94f1944e7 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Wed, 3 Jun 2020 08:49:32 -0400 Subject: [PATCH 19/39] temp checkin --- packages/firestore/src/core/sync_engine.ts | 26 ++++++++++++++++++++ packages/firestore/src/util/bundle_reader.ts | 4 +++ 2 files changed, 30 insertions(+) diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 0204f403e50..236efd80658 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -74,6 +74,7 @@ import { import { ViewSnapshot } from './view_snapshot'; import { AsyncQueue, wrapInUserErrorIfRecoverable } from '../util/async_queue'; import { TransactionRunner } from './transaction_runner'; +import { BundleReader } from '../util/bundle_reader'; const LOG_TAG = 'SyncEngine'; @@ -439,6 +440,31 @@ export class SyncEngine implements RemoteSyncer { } } + async loadBundle(bundleReader: BundleReader): Promise { + this.assertSubscribed('loadBundle()'); + const metadata = await bundleReader.getMetadata(); + + const skip = await this.localStore.isNewerBundleLoaded(metadata); + if (skip) { + return bundleReader.close(); + } + + while (true) { + const e = await bundleReader.nextElement(); + if (!e) { + break; + } + if (e.payload.namedQuery) { + await this.localStore.saveNamedQuery(e.payload.namedQuery); + } + if (e.payload.documentMetadata) { + await this.localStore.applyBundledDocuments(); + } + } + + return this.localStore.saveBundle(metadata); + } + /** * Applies an OnlineState change to the sync engine and notifies any views of * the change. diff --git a/packages/firestore/src/util/bundle_reader.ts b/packages/firestore/src/util/bundle_reader.ts index aff7a25c77d..d7da86a13c7 100644 --- a/packages/firestore/src/util/bundle_reader.ts +++ b/packages/firestore/src/util/bundle_reader.ts @@ -84,6 +84,10 @@ export class BundleReader { ); } + close(): Promise { + return this.reader.cancel(); + } + /** * Returns the metadata of the bundle. */ From 17ba434fa246fe3630071b658a59ddd25c91f6fb Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 4 Jun 2020 13:03:46 -0400 Subject: [PATCH 20/39] Implement without async/await --- packages/firestore/src/core/bundle.ts | 213 ++++++++++++++++++ .../firestore/src/core/firestore_client.ts | 6 + packages/firestore/src/core/sync_engine.ts | 64 ++++-- 3 files changed, 265 insertions(+), 18 deletions(-) diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index bcd8bedcfe0..0607ec52c79 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -23,6 +23,10 @@ import * as api from '../protos/firestore_proto_api'; import { DocumentKey } from '../model/document_key'; import { MaybeDocument, NoDocument } from '../model/document'; import { debugAssert } from '../util/assert'; +import { LocalStore } from '../local/local_store'; +import { SizedBundleElement } from '../util/bundle_reader'; +import { MaybeDocumentMap } from '../model/collections'; +import { Deferred } from '../util/promise'; /** * Represents a Firestore bundle saved by the SDK in its local storage. @@ -74,3 +78,212 @@ export class BundleConverter { return this.serializer.fromVersion(time); } } + +export interface LoadBundleTaskProgress { + documentsLoaded: number; + totalDocuments: number; + bytesLoaded: number; + totalBytes: number; + taskState: TaskState; +} +export type TaskState = 'Error' | 'Running' | 'Success'; + +export interface LoadBundleTask { + onProgress( + next?: (a: LoadBundleTaskProgress) => any, + error?: (a: Error) => any, + complete?: () => void + ): Promise; + + then( + onFulfilled?: (a: LoadBundleTaskProgress) => any, + onRejected?: (a: Error) => any + ): Promise; + + catch(onRejected: (a: Error) => any): Promise; +} + +export class LoadBundleTaskImpl implements LoadBundleTask { + private progressResolver = new Deferred(); + private progressNext?: (a: LoadBundleTaskProgress) => any; + private progressError?: (a: Error) => any; + private progressComplete?: () => any; + + private promiseResolver = new Deferred(); + private promiseFulfilled?: (a: LoadBundleTaskProgress) => any; + private promiseRejected?: (a: Error) => any; + + private lastProgress: LoadBundleTaskProgress = { + taskState: 'Running', + totalBytes: 0, + totalDocuments: 0, + bytesLoaded: 0, + documentsLoaded: 0 + }; + + onProgress( + next?: (a: LoadBundleTaskProgress) => any, + error?: (a: Error) => any, + complete?: () => void + ): Promise { + this.progressNext = next; + this.progressError = error; + this.progressComplete = complete; + return this.progressResolver.promise; + } + + catch(onRejected: (a: Error) => any): Promise { + this.promiseRejected = onRejected; + return this.promiseResolver.promise; + } + + then( + onFulfilled?: (a: LoadBundleTaskProgress) => any, + onRejected?: (a: Error) => any + ): Promise { + this.promiseFulfilled = onFulfilled; + this.promiseRejected = onRejected; + return this.promiseResolver.promise; + } + + completeWith(progress: LoadBundleTaskProgress): void { + let result; + if (this.progressComplete) { + result = this.progressComplete(); + } + this.progressResolver.resolve(result); + + result = undefined; + if (this.promiseFulfilled) { + result = this.promiseFulfilled(progress); + } + this.promiseResolver.resolve(result); + } + + failedWith(error: Error): void { + if (this.progressNext) { + this.lastProgress!.taskState = 'Error'; + this.progressNext(this.lastProgress); + } + + let result; + if (this.progressError) { + result = this.progressError(error); + } + this.progressResolver.reject(result); + + result = undefined; + if (this.promiseRejected) { + this.promiseRejected(error); + } + this.promiseResolver.reject(result); + } + + updateProgress(progress: LoadBundleTaskProgress): void { + this.lastProgress = progress; + if (this.progressNext) { + this.progressNext(progress); + } + } +} + +export class LoadResult { + constructor( + readonly progress: LoadBundleTaskProgress, + readonly changedDocs?: MaybeDocumentMap + ) {} +} + +export class BundleLoader { + private progress: LoadBundleTaskProgress; + private step = 0.01; + private queries: bundleProto.NamedQuery[] = []; + private documents: BundledDocuments = []; + private bytesIncrement = 0; + private documentsIncrement = 0; + private unpairedDocumentMetadata: bundleProto.BundledDocumentMetadata | null = null; + + constructor( + private metadata: bundleProto.BundleMetadata, + private localStore: LocalStore + ) { + this.progress = { + documentsLoaded: 0, + totalDocuments: metadata.totalDocuments!, + bytesLoaded: 0, + totalBytes: metadata.totalBytes!, + taskState: 'Running' + }; + } + + addSizedElement(element: SizedBundleElement): Promise { + debugAssert(!element.isBundleMetadata(), 'Unexpected bundle metadata.'); + + this.bytesIncrement += element.byteLength; + if (element.payload.namedQuery) { + this.queries.push(element.payload.namedQuery); + } + + if (element.payload.documentMetadata) { + if (element.payload.documentMetadata.exists) { + this.unpairedDocumentMetadata = element.payload.documentMetadata; + } else { + this.documents.push([element.payload.documentMetadata, undefined]); + this.documentsIncrement += 1; + } + } + + if (element.payload.document) { + debugAssert( + !this.unpairedDocumentMetadata, + 'Unexpected document when no paring metadata is found' + ); + this.documents.push([ + this.unpairedDocumentMetadata!, + element.payload.document + ]); + this.documentsIncrement += 1; + this.unpairedDocumentMetadata = null; + } + + return this.saveAndReportProgress(); + } + + private async saveAndReportProgress(): Promise { + if ( + this.unpairedDocumentMetadata || + (this.documentsIncrement < this.progress.totalDocuments * this.step && + this.bytesIncrement < this.progress.totalBytes * this.step) + ) { + return null; + } + + for (const q of this.queries) { + await this.localStore.saveNamedQuery(q); + } + + let changedDocs; + if (this.documents.length > 0) { + changedDocs = await this.localStore.applyBundledDocuments(this.documents); + } + + this.progress.bytesLoaded += this.bytesIncrement; + this.progress.documentsLoaded += this.documentsIncrement; + this.bytesIncrement = 0; + this.documentsIncrement = 0; + this.queries = []; + this.documents = []; + + return new LoadResult({ ...this.progress }, changedDocs); + } + + async complete(): Promise { + debugAssert( + this.queries.length === 0 && this.documents.length === 0, + 'There are more items needs to be saved but complete() is called.' + ); + this.progress.taskState = 'Success'; + + return this.progress; + } +} diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index 42fbe0173f5..166c4105b34 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -48,6 +48,8 @@ import { ComponentProvider, MemoryComponentProvider } from './component_provider'; +import { BundleReader } from '../util/bundle_reader'; +import { LoadBundleTask } from './bundle'; const LOG_TAG = 'FirestoreClient'; const MAX_CONCURRENT_LIMBO_RESOLUTIONS = 100; @@ -521,4 +523,8 @@ export class FirestoreClient { }); return deferred.promise; } + + loadBundle(bundleReader: BundleReader): LoadBundleTask { + this.verifyNotTerminated(); + } } diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 236efd80658..967f335836f 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -75,6 +75,8 @@ import { ViewSnapshot } from './view_snapshot'; import { AsyncQueue, wrapInUserErrorIfRecoverable } from '../util/async_queue'; import { TransactionRunner } from './transaction_runner'; import { BundleReader } from '../util/bundle_reader'; +import { BundleLoader, LoadBundleTask, LoadBundleTaskImpl } from './bundle'; +import { BundleMetadata } from '../protos/firestore_bundle_proto'; const LOG_TAG = 'SyncEngine'; @@ -440,29 +442,55 @@ export class SyncEngine implements RemoteSyncer { } } - async loadBundle(bundleReader: BundleReader): Promise { + loadBundle(bundleReader: BundleReader, task: LoadBundleTaskImpl): void { this.assertSubscribed('loadBundle()'); - const metadata = await bundleReader.getMetadata(); - - const skip = await this.localStore.isNewerBundleLoaded(metadata); - if (skip) { - return bundleReader.close(); - } + let metadata: BundleMetadata; + let loader: BundleLoader; + bundleReader + .getMetadata() + .then(m => { + metadata = m; + return this.localStore.isNewerBundleLoaded(metadata); + }) + .then(skip => { + if (skip) { + return bundleReader.close().then(() => { + // task.completeWith({}); + }); + } else { + loader = new BundleLoader(metadata, this.localStore); + return this.loadRestElements(loader, bundleReader, task) + .then(() => this.localStore.saveBundle(metadata)) + .then(() => loader.complete()) + .then(progress => task.completeWith(progress)); + } + }) + .catch(reason => { + task.failedWith(new Error(reason)); + }); + } - while (true) { - const e = await bundleReader.nextElement(); - if (!e) { - break; - } - if (e.payload.namedQuery) { - await this.localStore.saveNamedQuery(e.payload.namedQuery); + private async loadRestElements( + loader: BundleLoader, + reader: BundleReader, + task: LoadBundleTaskImpl + ): Promise { + const element = await reader.nextElement(); + if (element) { + debugAssert( + !element.payload.metadata, + 'Unexpected BundleMetadata element.' + ); + const result = await loader.addSizedElement(element); + if (result) { + task.updateProgress(result.progress); } - if (e.payload.documentMetadata) { - await this.localStore.applyBundledDocuments(); + if (result && result.changedDocs) { + this.emitNewSnapsAndNotifyLocalStore(result.changedDocs); } - } - return this.localStore.saveBundle(metadata); + return this.loadRestElements(loader, reader, task); + } } /** From f808d8d7729896e9fff56d9e6f0e849bb8c67d1c Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 4 Jun 2020 13:39:41 -0400 Subject: [PATCH 21/39] Major code complete. --- packages/firestore-types/index.d.ts | 5 ++ packages/firestore/src/api/database.ts | 7 ++ packages/firestore/src/core/bundle.ts | 2 + .../firestore/src/core/firestore_client.ts | 14 +++- packages/firestore/src/core/sync_engine.ts | 66 ++++++++++--------- 5 files changed, 60 insertions(+), 34 deletions(-) diff --git a/packages/firestore-types/index.d.ts b/packages/firestore-types/index.d.ts index 4b5e745f70a..234f4735c80 100644 --- a/packages/firestore-types/index.d.ts +++ b/packages/firestore-types/index.d.ts @@ -16,6 +16,7 @@ */ import { FirebaseApp, FirebaseNamespace } from '@firebase/app-types'; +import { LoadBundleTask } from '../firestore/src/core/bundle'; export type DocumentData = { [field: string]: any }; @@ -85,6 +86,10 @@ export class FirebaseFirestore { terminate(): Promise; + loadBundle( + bundleData: ArrayBuffer | ReadableStream /*| string */ + ): LoadBundleTask; + INTERNAL: { delete: () => Promise }; } diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index 01b58340ca1..ee38a3747ca 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -92,6 +92,7 @@ import { fieldPathFromArgument, UserDataReader } from './user_data_reader'; import { UserDataWriter } from './user_data_writer'; import { FirebaseAuthInternalName } from '@firebase/auth-interop-types'; import { Provider } from '@firebase/component'; +import { LoadBundleTask } from '../core/bundle'; // settings() defaults: const DEFAULT_HOST = 'firestore.googleapis.com'; @@ -476,6 +477,12 @@ export class Firestore implements firestore.FirebaseFirestore, FirebaseService { }; } + loadBundle( + bundleData: ArrayBuffer | ReadableStream /*| string */ + ): LoadBundleTask { + return this._firestoreClient!.loadBundle(bundleData); + } + ensureClientConfigured(): FirestoreClient { if (!this._firestoreClient) { // Kick off starting the client but don't actually wait for it. diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index 0607ec52c79..d4901f40c73 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -88,6 +88,7 @@ export interface LoadBundleTaskProgress { } export type TaskState = 'Error' | 'Running' | 'Success'; +/* eslint-disable @typescript-eslint/no-explicit-any */ export interface LoadBundleTask { onProgress( next?: (a: LoadBundleTaskProgress) => any, @@ -186,6 +187,7 @@ export class LoadBundleTaskImpl implements LoadBundleTask { } } } +/* eslint-enable @typescript-eslint/no-explicit-any */ export class LoadResult { constructor( diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index 166c4105b34..0723ec9c7bc 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -49,7 +49,7 @@ import { MemoryComponentProvider } from './component_provider'; import { BundleReader } from '../util/bundle_reader'; -import { LoadBundleTask } from './bundle'; +import { LoadBundleTask, LoadBundleTaskImpl } from './bundle'; const LOG_TAG = 'FirestoreClient'; const MAX_CONCURRENT_LIMBO_RESOLUTIONS = 100; @@ -524,7 +524,17 @@ export class FirestoreClient { return deferred.promise; } - loadBundle(bundleReader: BundleReader): LoadBundleTask { + loadBundle( + data: ReadableStream | ArrayBuffer | Uint8Array + ): LoadBundleTask { this.verifyNotTerminated(); + + const reader = new BundleReader(data); + const task = new LoadBundleTaskImpl(); + this.asyncQueue.enqueueAndForget(() => { + return this.syncEngine.loadBundle(reader, task); + }); + + return task; } } diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 967f335836f..c6956805017 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -75,8 +75,7 @@ import { ViewSnapshot } from './view_snapshot'; import { AsyncQueue, wrapInUserErrorIfRecoverable } from '../util/async_queue'; import { TransactionRunner } from './transaction_runner'; import { BundleReader } from '../util/bundle_reader'; -import { BundleLoader, LoadBundleTask, LoadBundleTaskImpl } from './bundle'; -import { BundleMetadata } from '../protos/firestore_bundle_proto'; +import { BundleLoader, LoadBundleTaskImpl } from './bundle'; const LOG_TAG = 'SyncEngine'; @@ -442,41 +441,38 @@ export class SyncEngine implements RemoteSyncer { } } - loadBundle(bundleReader: BundleReader, task: LoadBundleTaskImpl): void { + loadBundle( + bundleReader: BundleReader, + task: LoadBundleTaskImpl + ): Promise { this.assertSubscribed('loadBundle()'); - let metadata: BundleMetadata; - let loader: BundleLoader; - bundleReader - .getMetadata() - .then(m => { - metadata = m; - return this.localStore.isNewerBundleLoaded(metadata); - }) - .then(skip => { - if (skip) { - return bundleReader.close().then(() => { - // task.completeWith({}); - }); - } else { - loader = new BundleLoader(metadata, this.localStore); - return this.loadRestElements(loader, bundleReader, task) - .then(() => this.localStore.saveBundle(metadata)) - .then(() => loader.complete()) - .then(progress => task.completeWith(progress)); - } - }) - .catch(reason => { - task.failedWith(new Error(reason)); - }); + + return this.loadBundleAsync(bundleReader, task).catch(reason => { + task.failedWith(new Error(reason)); + }); } - private async loadRestElements( - loader: BundleLoader, + private async loadBundleAsync( reader: BundleReader, task: LoadBundleTaskImpl ): Promise { - const element = await reader.nextElement(); - if (element) { + const metadata = await reader.getMetadata(); + const skip = await this.localStore.isNewerBundleLoaded(metadata); + if (skip) { + await reader.close(); + task.completeWith({ + taskState: 'Success', + documentsLoaded: 0, + bytesLoaded: 0, + totalDocuments: metadata.totalDocuments!, + totalBytes: metadata.totalBytes! + }); + return; + } + + const loader = new BundleLoader(metadata, this.localStore); + let element = await reader.nextElement(); + while (element) { debugAssert( !element.payload.metadata, 'Unexpected BundleMetadata element.' @@ -486,11 +482,17 @@ export class SyncEngine implements RemoteSyncer { task.updateProgress(result.progress); } if (result && result.changedDocs) { + // eslint-disable-next-line @typescript-eslint/no-floating-promises this.emitNewSnapsAndNotifyLocalStore(result.changedDocs); } - return this.loadRestElements(loader, reader, task); + element = await reader.nextElement(); } + + await this.localStore.saveBundle(metadata); + + const completeProgress = await loader.complete(); + task.completeWith(completeProgress); } /** From db1d864a76d1a390c15f452c489429bd9503f13e Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Sat, 6 Jun 2020 18:17:54 -0400 Subject: [PATCH 22/39] Integration tests added. --- packages/firestore/src/api/database.ts | 1 + packages/firestore/src/core/bundle.ts | 40 ++- packages/firestore/src/core/sync_engine.ts | 12 +- packages/firestore/src/local/local_store.ts | 2 +- .../test/integration/api/bundle.test.ts | 227 ++++++++++++++++++ .../test/unit/local/bundle_cache.test.ts | 2 +- .../test/unit/specs/spec_test_components.ts | 4 +- .../firestore/test/unit/util/bundle.test.ts | 123 ++-------- packages/firestore/test/util/bundle_data.ts | 213 ++++++++++++++++ 9 files changed, 495 insertions(+), 129 deletions(-) create mode 100644 packages/firestore/test/integration/api/bundle.test.ts create mode 100644 packages/firestore/test/util/bundle_data.ts diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index ee38a3747ca..3a21ac61376 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -480,6 +480,7 @@ export class Firestore implements firestore.FirebaseFirestore, FirebaseService { loadBundle( bundleData: ArrayBuffer | ReadableStream /*| string */ ): LoadBundleTask { + this.ensureClientConfigured(); return this._firestoreClient!.loadBundle(bundleData); } diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index d4901f40c73..4f5995b8650 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -27,6 +27,7 @@ import { LocalStore } from '../local/local_store'; import { SizedBundleElement } from '../util/bundle_reader'; import { MaybeDocumentMap } from '../model/collections'; import { Deferred } from '../util/promise'; +import { BundleMetadata } from '../protos/firestore_bundle_proto'; /** * Represents a Firestore bundle saved by the SDK in its local storage. @@ -88,13 +89,26 @@ export interface LoadBundleTaskProgress { } export type TaskState = 'Error' | 'Running' | 'Success'; +export function initialProgress( + state: TaskState, + metadata: BundleMetadata +): LoadBundleTaskProgress { + return { + taskState: state, + documentsLoaded: state === 'Success' ? metadata.totalDocuments! : 0, + bytesLoaded: state === 'Success' ? metadata.totalBytes! : 0, + totalDocuments: metadata.totalDocuments!, + totalBytes: metadata.totalBytes! + }; +} + /* eslint-disable @typescript-eslint/no-explicit-any */ export interface LoadBundleTask { onProgress( - next?: (a: LoadBundleTaskProgress) => any, - error?: (a: Error) => any, - complete?: () => void - ): Promise; + next?: (progress: LoadBundleTaskProgress) => any, + error?: (error: Error) => any, + complete?: (progress?: LoadBundleTaskProgress) => any + ): Promise; then( onFulfilled?: (a: LoadBundleTaskProgress) => any, @@ -106,9 +120,9 @@ export interface LoadBundleTask { export class LoadBundleTaskImpl implements LoadBundleTask { private progressResolver = new Deferred(); - private progressNext?: (a: LoadBundleTaskProgress) => any; - private progressError?: (a: Error) => any; - private progressComplete?: () => any; + private progressNext?: (progress: LoadBundleTaskProgress) => any; + private progressError?: (err: Error) => any; + private progressComplete?: (progress?: LoadBundleTaskProgress) => any; private promiseResolver = new Deferred(); private promiseFulfilled?: (a: LoadBundleTaskProgress) => any; @@ -123,9 +137,9 @@ export class LoadBundleTaskImpl implements LoadBundleTask { }; onProgress( - next?: (a: LoadBundleTaskProgress) => any, - error?: (a: Error) => any, - complete?: () => void + next?: (progress: LoadBundleTaskProgress) => any, + error?: (err: Error) => any, + complete?: (progress?: LoadBundleTaskProgress) => void ): Promise { this.progressNext = next; this.progressError = error; @@ -150,7 +164,7 @@ export class LoadBundleTaskImpl implements LoadBundleTask { completeWith(progress: LoadBundleTaskProgress): void { let result; if (this.progressComplete) { - result = this.progressComplete(); + result = this.progressComplete(progress); } this.progressResolver.resolve(result); @@ -237,8 +251,8 @@ export class BundleLoader { if (element.payload.document) { debugAssert( - !this.unpairedDocumentMetadata, - 'Unexpected document when no paring metadata is found' + !!this.unpairedDocumentMetadata, + 'Unexpected document when no pairing metadata is found' ); this.documents.push([ this.unpairedDocumentMetadata!, diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index c6956805017..d40a35f5b06 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -75,7 +75,7 @@ import { ViewSnapshot } from './view_snapshot'; import { AsyncQueue, wrapInUserErrorIfRecoverable } from '../util/async_queue'; import { TransactionRunner } from './transaction_runner'; import { BundleReader } from '../util/bundle_reader'; -import { BundleLoader, LoadBundleTaskImpl } from './bundle'; +import { BundleLoader, initialProgress, LoadBundleTaskImpl } from './bundle'; const LOG_TAG = 'SyncEngine'; @@ -460,16 +460,12 @@ export class SyncEngine implements RemoteSyncer { const skip = await this.localStore.isNewerBundleLoaded(metadata); if (skip) { await reader.close(); - task.completeWith({ - taskState: 'Success', - documentsLoaded: 0, - bytesLoaded: 0, - totalDocuments: metadata.totalDocuments!, - totalBytes: metadata.totalBytes! - }); + task.completeWith(initialProgress('Success', metadata)); return; } + task.updateProgress(initialProgress('Running', metadata)); + const loader = new BundleLoader(metadata, this.localStore); let element = await reader.nextElement(); while (element) { diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index c2005122726..5468bb9e911 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -755,7 +755,7 @@ export class LocalStore { return this.bundleCache.getBundle(transaction, bundleMetadata.id!); }) .then(cached => { - return !!cached && cached!.createTime!.compareTo(currentReadTime) > 0; + return !!cached && cached!.createTime!.compareTo(currentReadTime) >= 0; }); } diff --git a/packages/firestore/test/integration/api/bundle.test.ts b/packages/firestore/test/integration/api/bundle.test.ts new file mode 100644 index 00000000000..e57a6ba50cb --- /dev/null +++ b/packages/firestore/test/integration/api/bundle.test.ts @@ -0,0 +1,227 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as firestore from '@firebase/firestore-types'; +import { expect } from 'chai'; +import { apiDescribe, toDataArray, withTestDb } from '../util/helpers'; +import { TestBundleBuilder } from '../../util/bundle_data'; +import { DatabaseId } from '../../../src/core/database_info'; +import { key } from '../../util/helpers'; +import { LoadBundleTaskProgress } from '../../../src/core/bundle'; +import { EventsAccumulator } from '../util/events_accumulator'; + +function verifySuccessProgress(p: LoadBundleTaskProgress): void { + expect(p.taskState).to.equal('Success'); + expect(p.bytesLoaded).to.be.equal(p.totalBytes); + expect(p.documentsLoaded).to.equal(p.totalDocuments); +} + +function verifyInProgress( + p: LoadBundleTaskProgress, + expectedDocuments: number +): void { + expect(p.taskState).to.equal('Running'); + expect(p.bytesLoaded).lte(p.totalBytes); + expect(p.documentsLoaded).lte(p.totalDocuments); + expect(p.documentsLoaded).to.equal(expectedDocuments); +} + +apiDescribe.only('Bundles', (persistence: boolean) => { + const encoder = new TextEncoder(); + const testDocs: { [key: string]: firestore.DocumentData } = { + a: { k: { stringValue: 'a' }, bar: { integerValue: 1 } }, + b: { k: { stringValue: 'b' }, bar: { integerValue: 2 } } + }; + + function bundleWithTestDocs( + db: firestore.FirebaseFirestore + ): TestBundleBuilder { + const a = key('coll-1/a'); + const b = key('coll-1/b'); + const builder = new TestBundleBuilder( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (db as any)._databaseId as DatabaseId + ); + builder.addDocumentMetadata(a, { seconds: 1000, nanos: 9999 }, true); + builder.addDocument( + a, + { seconds: 1, nanos: 9 }, + { seconds: 1, nanos: 9 }, + testDocs.a + ); + builder.addDocumentMetadata(b, { seconds: 1000, nanos: 9999 }, true); + builder.addDocument( + b, + { seconds: 1, nanos: 9 }, + { seconds: 1, nanos: 9 }, + testDocs.b + ); + + return builder; + } + + it('load with documents only with on progress and promise interface.', () => { + return withTestDb(persistence, async db => { + const builder = bundleWithTestDocs(db); + + const progresses: LoadBundleTaskProgress[] = []; + let completeProgress: LoadBundleTaskProgress, + fulfillProgress: LoadBundleTaskProgress; + await db + .loadBundle( + encoder.encode( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ) + ) + .onProgress( + progress => { + progresses.push(progress); + }, + err => { + throw err; + }, + progress => { + completeProgress = progress!; + return progress; + } + ) + .then(progress => { + fulfillProgress = progress; + }) + .catch(err => { + throw err; + }); + + verifySuccessProgress(completeProgress!); + verifySuccessProgress(fulfillProgress!); + expect(progresses.length).to.equal(3); + verifyInProgress(progresses[0], 0); + verifyInProgress(progresses[1], 1); + verifyInProgress(progresses[2], 2); + + // Read from cache. These documents do not exist in backend, so they can + // only be read from cache. + const snap = await db.collection('coll-1').get({ source: 'cache' }); + expect(toDataArray(snap)).to.deep.equal([ + { k: 'a', bar: 1 }, + { k: 'b', bar: 2 } + ]); + }); + }); + + it('load with documents with promise interface.', () => { + return withTestDb(persistence, async db => { + const builder = bundleWithTestDocs(db); + + let fulfillProgress: LoadBundleTaskProgress; + await db + .loadBundle( + encoder.encode( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ) + ) + .then(progress => { + fulfillProgress = progress; + }) + .catch(err => { + throw err; + }); + + verifySuccessProgress(fulfillProgress!); + + // Read from cache. These documents do not exist in backend, so they can + // only be read from cache. + const snap = await db.collection('coll-1').get({ source: 'cache' }); + expect(toDataArray(snap)).to.deep.equal([ + { k: 'a', bar: 1 }, + { k: 'b', bar: 2 } + ]); + }); + }); + + it('load for a second time skips.', () => { + return withTestDb(persistence, async db => { + const builder = bundleWithTestDocs(db); + + await db.loadBundle( + encoder.encode( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ) + ); + + let completeProgress: LoadBundleTaskProgress; + const progresses: LoadBundleTaskProgress[] = []; + await db + .loadBundle( + encoder.encode( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ) + ) + .onProgress( + progress => { + progresses.push(progress); + }, + error => {}, + progress => { + completeProgress = progress!; + } + ); + + // No loading actually happened in the second `loadBundle` call. + expect(progresses).to.be.empty; + verifySuccessProgress(completeProgress!); + + // Read from cache. These documents do not exist in backend, so they can + // only be read from cache. + const snap = await db.collection('coll-1').get({ source: 'cache' }); + expect(toDataArray(snap)).to.deep.equal([ + { k: 'a', bar: 1 }, + { k: 'b', bar: 2 } + ]); + }); + }); + + it('load with documents already pulled from backend.', () => { + return withTestDb(persistence, async db => { + await db.doc('coll-1/a').set({ k: 'a', bar: 0 }); + await db.doc('coll-1/b').set({ k: 'b', bar: 0 }); + + const accumulator = new EventsAccumulator(); + db.collection('coll-1').onSnapshot(accumulator.storeEvent); + await accumulator.awaitEvent(); + + const builder = bundleWithTestDocs(db); + const progress = await db.loadBundle( + encoder.encode( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ) + ); + + verifySuccessProgress(progress); + // The test bundle is holding ancient documents, so no events are + // generated as a result. The case where a bundle has newer doc than + // cache can only be tested in spec tests. + await accumulator.assertNoAdditionalEvents(); + + const snap = await db.collection('coll-1').get(); + expect(toDataArray(snap)).to.deep.equal([ + { k: 'a', bar: 0 }, + { k: 'b', bar: 0 } + ]); + }); + }); +}); diff --git a/packages/firestore/test/unit/local/bundle_cache.test.ts b/packages/firestore/test/unit/local/bundle_cache.test.ts index 11c7b1c7e67..c3dea781bb9 100644 --- a/packages/firestore/test/unit/local/bundle_cache.test.ts +++ b/packages/firestore/test/unit/local/bundle_cache.test.ts @@ -78,7 +78,7 @@ function genericBundleCacheTests(cacheFn: () => TestBundleCache): void { expectedQuery: Query, expectedReadSeconds: number, expectedReadNanos: number - ):void { + ): void { expect(actual.name).to.equal(expectedName); expect(actual.query.isEqual(expectedQuery)).to.be.true; expect( diff --git a/packages/firestore/test/unit/specs/spec_test_components.ts b/packages/firestore/test/unit/specs/spec_test_components.ts index 1da4d480d8c..2bf5b0883d6 100644 --- a/packages/firestore/test/unit/specs/spec_test_components.ts +++ b/packages/firestore/test/unit/specs/spec_test_components.ts @@ -51,9 +51,7 @@ import { ViewSnapshot } from '../../../src/core/view_snapshot'; import { Query } from '../../../src/core/query'; import { Mutation } from '../../../src/model/mutation'; import { expect } from 'chai'; -import { - JSON_SERIALIZER -} from '../local/persistence_test_helpers'; +import { JSON_SERIALIZER } from '../local/persistence_test_helpers'; /** * A test-only MemoryPersistence implementation that is able to inject diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index e915d2e5dfc..83aeea0c09a 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -19,12 +19,29 @@ import { BundleReader, SizedBundleElement } from '../../../src/util/bundle_reader'; -import { BundleElement } from '../../../src/protos/firestore_bundle_proto'; import { isNode } from '../../util/test_platform'; import { PlatformSupport, toByteStreamReader } from '../../../src/platform/platform'; +import { + doc1String, + doc1MetaString, + doc1Meta, + noDocMetaString, + noDocMeta, + doc2MetaString, + doc2Meta, + limitQueryString, + limitQuery, + limitToLastQuery, + limitToLastQueryString, + meta, + metaString, + doc2String, + doc1, + doc2 +} from '../../util/bundle_data'; /** * Create a `ReadableStream` from a string. @@ -51,12 +68,6 @@ export function readableStreamFromString( }); } -function lengthPrefixedString(o: {}): string { - const str = JSON.stringify(o); - const l = new TextEncoder().encode(str).byteLength; - return `${l}${str}`; -} - // Testing readableStreamFromString() is working as expected. // eslint-disable-next-line no-restricted-properties (isNode() ? describe.skip : describe)('readableStreamFromString()', () => { @@ -84,6 +95,8 @@ function lengthPrefixedString(o: {}): string { }); }); +const encoder = new TextEncoder(); + describe('Bundle ', () => { genericBundleReadingTests(1); genericBundleReadingTests(4); @@ -109,102 +122,6 @@ function genericBundleReadingTests(bytesPerRead: number): void { } } - const encoder = new TextEncoder(); - // Setting up test data. - const meta: BundleElement = { - metadata: { - id: 'test-bundle', - createTime: { seconds: 1577836805, nanos: 6 }, - version: 1, - totalDocuments: 1, - totalBytes: 416 - } - }; - const metaString = lengthPrefixedString(meta); - - const doc1Meta: BundleElement = { - documentMetadata: { - name: - 'projects/test-project/databases/(default)/documents/collectionId/doc1', - readTime: { seconds: 5, nanos: 6 }, - exists: true - } - }; - const doc1MetaString = lengthPrefixedString(doc1Meta); - const doc1: BundleElement = { - document: { - name: - 'projects/test-project/databases/(default)/documents/collectionId/doc1', - createTime: { seconds: 1, nanos: 2000000 }, - updateTime: { seconds: 3, nanos: 4000 }, - fields: { foo: { stringValue: 'value' }, bar: { integerValue: -42 } } - } - }; - const doc1String = lengthPrefixedString(doc1); - - const doc2Meta: BundleElement = { - documentMetadata: { - name: - 'projects/test-project/databases/(default)/documents/collectionId/doc2', - readTime: { seconds: 5, nanos: 6 }, - exists: true - } - }; - const doc2MetaString = lengthPrefixedString(doc2Meta); - const doc2: BundleElement = { - document: { - name: - 'projects/test-project/databases/(default)/documents/collectionId/doc2', - createTime: { seconds: 1, nanos: 2000000 }, - updateTime: { seconds: 3, nanos: 4000 }, - fields: { foo: { stringValue: 'value1' }, bar: { integerValue: 42 } } - } - }; - const doc2String = lengthPrefixedString(doc2); - - const noDocMeta: BundleElement = { - documentMetadata: { - name: - 'projects/test-project/databases/(default)/documents/collectionId/nodoc', - readTime: { seconds: 5, nanos: 6 }, - exists: false - } - }; - const noDocMetaString = lengthPrefixedString(noDocMeta); - - const limitQuery: BundleElement = { - namedQuery: { - name: 'limitQuery', - bundledQuery: { - parent: 'projects/fireeats-97d5e/databases/(default)/documents', - structuredQuery: { - from: [{ collectionId: 'node_3.7.5_7Li7XoCjutvNxwD0tpo9' }], - orderBy: [{ field: { fieldPath: 'sort' }, direction: 'DESCENDING' }], - limit: { 'value': 1 } - }, - limitType: 'FIRST' - }, - readTime: { 'seconds': 1590011379, 'nanos': 191164000 } - } - }; - const limitQueryString = lengthPrefixedString(limitQuery); - const limitToLastQuery: BundleElement = { - namedQuery: { - name: 'limitToLastQuery', - bundledQuery: { - parent: 'projects/fireeats-97d5e/databases/(default)/documents', - structuredQuery: { - from: [{ collectionId: 'node_3.7.5_7Li7XoCjutvNxwD0tpo9' }], - orderBy: [{ field: { fieldPath: 'sort' }, direction: 'ASCENDING' }], - limit: { 'value': 1 } - }, - limitType: 'LAST' - }, - readTime: { 'seconds': 1590011379, 'nanos': 543063000 } - } - }; - const limitToLastQueryString = lengthPrefixedString(limitToLastQuery); - async function getAllElements( bundle: BundleReader ): Promise { diff --git a/packages/firestore/test/util/bundle_data.ts b/packages/firestore/test/util/bundle_data.ts new file mode 100644 index 00000000000..37ea56e9662 --- /dev/null +++ b/packages/firestore/test/util/bundle_data.ts @@ -0,0 +1,213 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + BundledQuery, + BundleElement +} from '../../src/protos/firestore_bundle_proto'; +import { DatabaseId } from '../../src/core/database_info'; +import * as api from '../../src/protos/firestore_proto_api'; +import { Value } from '../../src/protos/firestore_proto_api'; +import { JsonProtoSerializer } from '../../src/remote/serializer'; +import { PlatformSupport } from '../../src/platform/platform'; +import { DocumentKey } from '../../src/model/document_key'; + +function lengthPrefixedString(o: {}): string { + const str = JSON.stringify(o); + const l = new TextEncoder().encode(str).byteLength; + return `${l}${str}`; +} + +export class TestBundleBuilder { + readonly elements: BundleElement[] = []; + private serializer: JsonProtoSerializer; + constructor(private databaseId: DatabaseId) { + this.serializer = PlatformSupport.getPlatform().newSerializer(databaseId); + } + + addDocumentMetadata( + docKey: DocumentKey, + readTime: api.Timestamp, + exists: boolean + ): TestBundleBuilder { + this.elements.push({ + documentMetadata: { + name: this.serializer.toName(docKey), + readTime, + exists + } + }); + return this; + } + addDocument( + docKey: DocumentKey, + createTime: api.Timestamp, + updateTime: api.Timestamp, + fields: api.ApiClientObjectMap + ): TestBundleBuilder { + this.elements.push({ + document: { + name: this.serializer.toName(docKey), + createTime, + updateTime, + fields + } + }); + return this; + } + addNamedQuery( + name: string, + readTime: api.Timestamp, + bundledQuery: BundledQuery + ): TestBundleBuilder { + this.elements.push({ namedQuery: { name, readTime, bundledQuery } }); + return this; + } + getMetadataElement( + id: string, + createTime: api.Timestamp, + version = 1 + ): BundleElement { + let totalDocuments = 0; + let totalBytes = 0; + const encoder = new TextEncoder(); + for (const element of this.elements) { + if (element.documentMetadata && !element.documentMetadata.exists) { + totalDocuments += 1; + } + if (element.document) { + totalDocuments += 1; + } + totalBytes += encoder.encode(lengthPrefixedString(element)).byteLength; + } + + return { + metadata: { + id, + createTime, + version, + totalDocuments, + totalBytes + } + }; + } + + build(id: string, createTime: api.Timestamp, version = 1): string { + let result = ''; + for (const element of this.elements) { + result += lengthPrefixedString(element); + } + return ( + lengthPrefixedString(this.getMetadataElement(id, createTime, version)) + + result + ); + } +} + +// Setting up test data. +export const meta: BundleElement = { + metadata: { + id: 'test-bundle', + createTime: { seconds: 1577836805, nanos: 6 }, + version: 1, + totalDocuments: 1, + totalBytes: 416 + } +}; +export const metaString = lengthPrefixedString(meta); + +export const doc1Meta: BundleElement = { + documentMetadata: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc1', + readTime: { seconds: 5, nanos: 6 }, + exists: true + } +}; +export const doc1MetaString = lengthPrefixedString(doc1Meta); +export const doc1: BundleElement = { + document: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc1', + createTime: { seconds: 1, nanos: 2000000 }, + updateTime: { seconds: 3, nanos: 4000 }, + fields: { foo: { stringValue: 'value' }, bar: { integerValue: -42 } } + } +}; +export const doc1String = lengthPrefixedString(doc1); + +export const doc2Meta: BundleElement = { + documentMetadata: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc2', + readTime: { seconds: 5, nanos: 6 }, + exists: true + } +}; +export const doc2MetaString = lengthPrefixedString(doc2Meta); +export const doc2: BundleElement = { + document: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/doc2', + createTime: { seconds: 1, nanos: 2000000 }, + updateTime: { seconds: 3, nanos: 4000 }, + fields: { foo: { stringValue: 'value1' }, bar: { integerValue: 42 } } + } +}; +export const doc2String = lengthPrefixedString(doc2); + +export const noDocMeta: BundleElement = { + documentMetadata: { + name: + 'projects/test-project/databases/(default)/documents/collectionId/nodoc', + readTime: { seconds: 5, nanos: 6 }, + exists: false + } +}; +export const noDocMetaString = lengthPrefixedString(noDocMeta); + +export const limitQuery: BundleElement = { + namedQuery: { + name: 'limitQuery', + bundledQuery: { + parent: 'projects/fireeats-97d5e/databases/(default)/documents', + structuredQuery: { + from: [{ collectionId: 'node_3.7.5_7Li7XoCjutvNxwD0tpo9' }], + orderBy: [{ field: { fieldPath: 'sort' }, direction: 'DESCENDING' }], + limit: { 'value': 1 } + }, + limitType: 'FIRST' + }, + readTime: { 'seconds': 1590011379, 'nanos': 191164000 } + } +}; +export const limitQueryString = lengthPrefixedString(limitQuery); +export const limitToLastQuery: BundleElement = { + namedQuery: { + name: 'limitToLastQuery', + bundledQuery: { + parent: 'projects/fireeats-97d5e/databases/(default)/documents', + structuredQuery: { + from: [{ collectionId: 'node_3.7.5_7Li7XoCjutvNxwD0tpo9' }], + orderBy: [{ field: { fieldPath: 'sort' }, direction: 'ASCENDING' }], + limit: { 'value': 1 } + }, + limitType: 'LAST' + }, + readTime: { 'seconds': 1590011379, 'nanos': 543063000 } + } +}; +export const limitToLastQueryString = lengthPrefixedString(limitToLastQuery); From 979ffd9fc7adc7ec1a174c15c2d8e1afb96fabff Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 9 Jun 2020 00:27:45 -0400 Subject: [PATCH 23/39] Added spec tests. Two TODO: 1. load from secondary tab. 2. don't fail async queue. --- packages/firestore/src/core/sync_engine.ts | 19 +- packages/firestore/src/local/local_store.ts | 2 +- .../test/integration/api/bundle.test.ts | 25 +- .../test/unit/specs/bundle_spec.test.ts | 354 ++++++++++++++++++ .../firestore/test/unit/specs/spec_builder.ts | 8 + .../test/unit/specs/spec_test_runner.ts | 16 + 6 files changed, 418 insertions(+), 6 deletions(-) create mode 100644 packages/firestore/test/unit/specs/bundle_spec.test.ts diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index d40a35f5b06..420551d65ef 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -447,9 +447,15 @@ export class SyncEngine implements RemoteSyncer { ): Promise { this.assertSubscribed('loadBundle()'); - return this.loadBundleAsync(bundleReader, task).catch(reason => { - task.failedWith(new Error(reason)); - }); + return ( + this.loadBundleAsync(bundleReader, task) + // TODO(wuandy): Loading a bundle will fail the entire SDK if there is + // an error. We really should have a way to run operations on async queue + // and not failing the rest if there is an error. + .catch(reason => { + task.failedWith(reason); + }) + ); } private async loadBundleAsync( @@ -1323,4 +1329,11 @@ export class MultiTabSyncEngine extends SyncEngine .catch(ignoreIfPrimaryLeaseLoss); } } + + loadBundle( + bundleReader: BundleReader, + task: LoadBundleTaskImpl + ): Promise { + return super.loadBundle(bundleReader, task); + } } diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index 5468bb9e911..6728bc49ae1 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -646,7 +646,7 @@ export class LocalStore { }); return this.persistence.runTransaction( 'Apply bundle documents', - 'readwrite-primary', + 'readwrite', txn => { return this.applyDocuments( documentBuffer, diff --git a/packages/firestore/test/integration/api/bundle.test.ts b/packages/firestore/test/integration/api/bundle.test.ts index e57a6ba50cb..10678ec2f00 100644 --- a/packages/firestore/test/integration/api/bundle.test.ts +++ b/packages/firestore/test/integration/api/bundle.test.ts @@ -17,7 +17,12 @@ import * as firestore from '@firebase/firestore-types'; import { expect } from 'chai'; -import { apiDescribe, toDataArray, withTestDb } from '../util/helpers'; +import { + apiDescribe, + toDataArray, + withAlternateTestDb, + withTestDb +} from '../util/helpers'; import { TestBundleBuilder } from '../../util/bundle_data'; import { DatabaseId } from '../../../src/core/database_info'; import { key } from '../../util/helpers'; @@ -40,7 +45,7 @@ function verifyInProgress( expect(p.documentsLoaded).to.equal(expectedDocuments); } -apiDescribe.only('Bundles', (persistence: boolean) => { +apiDescribe('Bundles', (persistence: boolean) => { const encoder = new TextEncoder(); const testDocs: { [key: string]: firestore.DocumentData } = { a: { k: { stringValue: 'a' }, bar: { integerValue: 1 } }, @@ -224,4 +229,20 @@ apiDescribe.only('Bundles', (persistence: boolean) => { ]); }); }); + + it('load with documents from other projects fails.', () => { + return withTestDb(persistence, async db => { + const builder = bundleWithTestDocs(db); + return withAlternateTestDb(persistence, async otherDb => { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + expect( + otherDb.loadBundle( + encoder.encode( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ) + ) + ).to.be.rejectedWith('Tried to deserialize key from different project'); + }); + }); + }); }); diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts new file mode 100644 index 00000000000..4c223179067 --- /dev/null +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -0,0 +1,354 @@ +/** + * @license + * Copyright 2017 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Query } from '../../../src/core/query'; +import { doc, path, TestSnapshotVersion, version } from '../../util/helpers'; + +import { describeSpec, specTest } from './describe_spec'; +import { client, spec } from './spec_builder'; +import { TestBundleBuilder } from '../../util/bundle_data'; +import { + JSON_SERIALIZER, + TEST_DATABASE_ID +} from '../local/persistence_test_helpers'; +import { DocumentKey } from '../../../src/model/document_key'; +import * as api from '../../../src/protos/firestore_proto_api'; +import { Value } from '../../../src/protos/firestore_proto_api'; +import { TimerId } from '../../../src/util/async_queue'; + +interface TestBundleDocument { + key: DocumentKey; + readTime: TestSnapshotVersion; + createTime?: TestSnapshotVersion; + updateTime?: TestSnapshotVersion; + content?: api.ApiClientObjectMap; +} +function bundleWithDocument(testDoc: TestBundleDocument): string { + const builder = new TestBundleBuilder(TEST_DATABASE_ID); + builder.addDocumentMetadata( + testDoc.key, + JSON_SERIALIZER.toVersion(version(testDoc.readTime)), + !!testDoc.createTime + ); + if (testDoc.createTime) { + builder.addDocument( + testDoc.key, + JSON_SERIALIZER.toVersion(version(testDoc.createTime)), + JSON_SERIALIZER.toVersion(version(testDoc.updateTime!)), + testDoc.content! + ); + } + return builder.build( + 'test-bundle', + JSON_SERIALIZER.toVersion(version(testDoc.readTime)) + ); +} + +describeSpec('Bundles:', [], () => { + specTest('Newer docs from bundles should overwrite cache.', [], () => { + const query1 = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + const docAChanged = doc('collection/a', 2999, { key: 'b' }); + + const bundleString = bundleWithDocument({ + key: docA.key, + readTime: 3000, + createTime: 1999, + updateTime: 2999, + content: { key: { stringValue: 'b' } } + }); + + return spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA) + .expectEvents(query1, { added: [docA] }) + .loadBundle(bundleString) + .expectEvents(query1, { modified: [docAChanged] }); + }); + + specTest('Newer deleted docs from bundles should delete cache.', [], () => { + const query1 = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + + const bundleString = bundleWithDocument({ + key: docA.key, + readTime: 3000 + }); + + return spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA) + .expectEvents(query1, { added: [docA] }) + .loadBundle(bundleString) + .expectEvents(query1, { removed: [docA] }); + }); + + specTest('Older deleted docs from bundles should do nothing.', [], () => { + const query1 = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + + const bundleString = bundleWithDocument({ + key: docA.key, + readTime: 999 + }); + + return ( + spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA) + .expectEvents(query1, { added: [docA] }) + // No events are expected here. + .loadBundle(bundleString) + ); + }); + + specTest( + 'Newer docs from bundles should raise snapshot only when watch catches up with acknowledged writes.', + [], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 250, { key: 'a' }); + + const bundleString1 = bundleWithDocument({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } + }); + + const bundleString2 = bundleWithDocument({ + key: docA.key, + readTime: 1001, + createTime: 250, + updateTime: 1001, + content: { key: { stringValue: 'fromBundle' } } + }); + return ( + spec() + .withGCEnabled(false) + .userListens(query) + .watchAcksFull(query, 250, docA) + .expectEvents(query, { + added: [doc('collection/a', 250, { key: 'a' })] + }) + .userPatches('collection/a', { key: 'patched' }) + .expectEvents(query, { + modified: [ + doc( + 'collection/a', + 250, + { key: 'patched' }, + { hasLocalMutations: true } + ) + ], + hasPendingWrites: true + }) + .writeAcks('collection/a', 1000) + // loading bundleString1 will not raise snapshots, because it is before + // the acknowledged mutation. + .loadBundle(bundleString1) + // loading bundleString2 will raise a snapshot, because it is after + // the acknowledged mutation. + .loadBundle(bundleString2) + .expectEvents(query, { + modified: [doc('collection/a', 1001, { key: 'fromBundle' })] + }) + ); + } + ); + + specTest( + 'Newer docs from bundles should keep not raise snapshot if there are unacknowledged writes.', + [], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 250, { key: 'a' }); + + const bundleString1 = bundleWithDocument({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } + }); + + const bundleString2 = bundleWithDocument({ + key: docA.key, + readTime: 1001, + createTime: 250, + updateTime: 1001, + content: { key: { stringValue: 'fromBundle' } } + }); + + return ( + spec() + .withGCEnabled(false) + .userListens(query) + .watchAcksFull(query, 250, docA) + .expectEvents(query, { + added: [doc('collection/a', 250, { key: 'a' })] + }) + .userPatches('collection/a', { key: 'patched' }) + .expectEvents(query, { + modified: [ + doc( + 'collection/a', + 250, + { key: 'patched' }, + { hasLocalMutations: true } + ) + ], + hasPendingWrites: true + }) + // Loading both bundles will not raise snapshots, because of the + // mutation is not acknowledged. + .loadBundle(bundleString1) + .loadBundle(bundleString2) + ); + } + ); + + specTest('Newer docs from bundles might lead to limbo doc.', [], () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); + const bundleString1 = bundleWithDocument({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } + }); + + return ( + spec() + .withGCEnabled(false) + .userListens(query) + .watchAcksFull(query, 250) + // Backend tells there is no such doc. + .expectEvents(query, {}) + // Bundle tells otherwise, leads to limbo resolution. + .loadBundle(bundleString1) + .expectEvents(query, { + added: [doc('collection/a', 500, { key: 'b' })], + fromCache: true + }) + .expectLimboDocs(docA.key) + ); + }); + + specTest( + 'Load from secondary clients and observe from primary.', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 250, { key: 'a' }); + const bundleString1 = bundleWithDocument({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } + }); + + return ( + client(0) + .userListens(query) + .watchAcksFull(query, 250, docA) + .expectEvents(query, { + added: [docA] + }) + .client(1) + // Bundle tells otherwise, leads to limbo resolution. + .loadBundle(bundleString1) + .client(0) + .becomeVisible() + // TODO(wuandy): Loading from secondary client does not notify other + // clients for now. We need to fix it and uncomment below. + // .expectEvents(query, { + // modified: [doc('collection/a', 500, { key: 'b' })], + // }) + ); + } + ); + + specTest('Load and observe from secondary clients.', ['multi-client'], () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 250, { key: 'a' }); + const bundleString1 = bundleWithDocument({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } + }); + + return client(0) + .userListens(query) + .watchAcksFull(query, 250, docA) + .expectEvents(query, { + added: [docA] + }) + .client(1) + .userListens(query) + .expectEvents(query, { + added: [docA] + }) + .loadBundle(bundleString1) + .expectEvents(query, { + modified: [doc('collection/a', 500, { key: 'b' })] + }); + }); + + specTest( + 'Load from primary client and observe from secondary.', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 250, { key: 'a' }); + const bundleString1 = bundleWithDocument({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } + }); + + return ( + client(0) + .userListens(query) + .watchAcksFull(query, 250, docA) + .expectEvents(query, { + added: [docA] + }) + .client(1) + .stealPrimaryLease() + .expectListen(query, 'resume-token-250') + // Bundle tells otherwise, leads to limbo resolution. + .loadBundle(bundleString1) + .client(0) + .runTimer(TimerId.ClientMetadataRefresh) + // Client 0 recovers from its lease loss and applies the updates from + // client 1 + .expectPrimaryState(false) + .expectEvents(query, { + modified: [doc('collection/a', 500, { key: 'b' })] + }) + ); + } + ); +}); diff --git a/packages/firestore/test/unit/specs/spec_builder.ts b/packages/firestore/test/unit/specs/spec_builder.ts index d5c0f2460ef..cf728c675a0 100644 --- a/packages/firestore/test/unit/specs/spec_builder.ts +++ b/packages/firestore/test/unit/specs/spec_builder.ts @@ -345,6 +345,14 @@ export class SpecBuilder { return this; } + loadBundle(bundleContent: string): this { + this.nextStep(); + this.currentStep = { + loadBundle: bundleContent + }; + return this; + } + // PORTING NOTE: Only used by web multi-tab tests. becomeHidden(): this { this.nextStep(); diff --git a/packages/firestore/test/unit/specs/spec_test_runner.ts b/packages/firestore/test/unit/specs/spec_test_runner.ts index 26c6a284636..1ad9c3cf685 100644 --- a/packages/firestore/test/unit/specs/spec_test_runner.ts +++ b/packages/firestore/test/unit/specs/spec_test_runner.ts @@ -113,6 +113,8 @@ import { SharedWriteTracker } from './spec_test_components'; import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; +import { BundleReader } from '../../../src/util/bundle_reader'; +import { LoadBundleTaskImpl } from '../../../src/core/bundle'; const ARBITRARY_SEQUENCE_NUMBER = 2; @@ -301,6 +303,8 @@ abstract class TestRunner { return this.doAddSnapshotsInSyncListener(); } else if ('removeSnapshotsInSyncListener' in step) { return this.doRemoveSnapshotsInSyncListener(); + } else if ('loadBundle' in step) { + return this.doLoadBundle(step.loadBundle!); } else if ('watchAck' in step) { return this.doWatchAck(step.watchAck!); } else if ('watchCurrent' in step) { @@ -438,6 +442,16 @@ abstract class TestRunner { return Promise.resolve(); } + private async doLoadBundle(bundle: string): Promise { + const reader = new BundleReader(new TextEncoder().encode(bundle)); + const task = new LoadBundleTaskImpl(); + this.queue.enqueueAndForget(() => { + return this.syncEngine.loadBundle(reader, task); + }); + + await task; + } + private doMutations(mutations: Mutation[]): Promise { const documentKeys = mutations.map(val => val.key.path.toString()); const syncEngineCallback = new Deferred(); @@ -1249,6 +1263,8 @@ export interface SpecStep { addSnapshotsInSyncListener?: true; /** Unlistens from a SnapshotsInSync event. */ removeSnapshotsInSyncListener?: true; + /** Loads a bundle from a string. */ + loadBundle?: string; /** Ack for a query in the watch stream */ watchAck?: SpecWatchAck; From f7ff495d7f130d8d44b1b0ae8c60541e241b22e2 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 9 Jun 2020 08:41:35 -0400 Subject: [PATCH 24/39] Add comments and move types to d.ts --- packages/firestore-types/index.d.ts | 28 ++++- packages/firestore/src/api/database.ts | 3 +- packages/firestore/src/core/bundle.ts | 117 ++++++++++-------- .../firestore/src/core/firestore_client.ts | 3 +- packages/firestore/src/core/sync_engine.ts | 18 +-- .../test/integration/api/bundle.test.ts | 17 ++- .../test/unit/specs/bundle_spec.test.ts | 2 +- packages/firestore/test/util/bundle_data.ts | 2 +- 8 files changed, 113 insertions(+), 77 deletions(-) diff --git a/packages/firestore-types/index.d.ts b/packages/firestore-types/index.d.ts index 234f4735c80..5e315240c63 100644 --- a/packages/firestore-types/index.d.ts +++ b/packages/firestore-types/index.d.ts @@ -16,7 +16,6 @@ */ import { FirebaseApp, FirebaseNamespace } from '@firebase/app-types'; -import { LoadBundleTask } from '../firestore/src/core/bundle'; export type DocumentData = { [field: string]: any }; @@ -87,12 +86,37 @@ export class FirebaseFirestore { terminate(): Promise; loadBundle( - bundleData: ArrayBuffer | ReadableStream /*| string */ + bundleData: ArrayBuffer | ReadableStream | string ): LoadBundleTask; INTERNAL: { delete: () => Promise }; } +export interface LoadBundleTask { + onProgress( + next?: (progress: LoadBundleTaskProgress) => any, + error?: (error: Error) => any, + complete?: (progress?: LoadBundleTaskProgress) => any + ): Promise; + + then( + onFulfilled?: (a: LoadBundleTaskProgress) => any, + onRejected?: (a: Error) => any + ): Promise; + + catch(onRejected: (a: Error) => any): Promise; +} + +export interface LoadBundleTaskProgress { + documentsLoaded: number; + totalDocuments: number; + bytesLoaded: number; + totalBytes: number; + taskState: TaskState; +} + +export type TaskState = 'Error' | 'Running' | 'Success'; + export class GeoPoint { constructor(latitude: number, longitude: number); diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index 3a21ac61376..06f70ecd7bc 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -92,7 +92,6 @@ import { fieldPathFromArgument, UserDataReader } from './user_data_reader'; import { UserDataWriter } from './user_data_writer'; import { FirebaseAuthInternalName } from '@firebase/auth-interop-types'; import { Provider } from '@firebase/component'; -import { LoadBundleTask } from '../core/bundle'; // settings() defaults: const DEFAULT_HOST = 'firestore.googleapis.com'; @@ -479,7 +478,7 @@ export class Firestore implements firestore.FirebaseFirestore, FirebaseService { loadBundle( bundleData: ArrayBuffer | ReadableStream /*| string */ - ): LoadBundleTask { + ): firestore.LoadBundleTask { this.ensureClientConfigured(); return this._firestoreClient!.loadBundle(bundleData); } diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index 4f5995b8650..c92e4f02e05 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -15,6 +15,8 @@ * limitations under the License. */ +import * as firestore from '@firebase/firestore-types'; + import { Query } from './query'; import { SnapshotVersion } from './snapshot_version'; import { JsonProtoSerializer } from '../remote/serializer'; @@ -80,19 +82,14 @@ export class BundleConverter { } } -export interface LoadBundleTaskProgress { - documentsLoaded: number; - totalDocuments: number; - bytesLoaded: number; - totalBytes: number; - taskState: TaskState; -} -export type TaskState = 'Error' | 'Running' | 'Success'; - +/** + * Returns a `LoadBundleTaskProgress` representing the first progress of + * loading a bundle. + */ export function initialProgress( - state: TaskState, + state: firestore.TaskState, metadata: BundleMetadata -): LoadBundleTaskProgress { +): firestore.LoadBundleTaskProgress { return { taskState: state, documentsLoaded: state === 'Success' ? metadata.totalDocuments! : 0, @@ -103,32 +100,21 @@ export function initialProgress( } /* eslint-disable @typescript-eslint/no-explicit-any */ -export interface LoadBundleTask { - onProgress( - next?: (progress: LoadBundleTaskProgress) => any, - error?: (error: Error) => any, - complete?: (progress?: LoadBundleTaskProgress) => any - ): Promise; - - then( - onFulfilled?: (a: LoadBundleTaskProgress) => any, - onRejected?: (a: Error) => any - ): Promise; - - catch(onRejected: (a: Error) => any): Promise; -} - -export class LoadBundleTaskImpl implements LoadBundleTask { +export class LoadBundleTaskImpl implements firestore.LoadBundleTask { private progressResolver = new Deferred(); - private progressNext?: (progress: LoadBundleTaskProgress) => any; + private progressNext?: (progress: firestore.LoadBundleTaskProgress) => any; private progressError?: (err: Error) => any; - private progressComplete?: (progress?: LoadBundleTaskProgress) => any; + private progressComplete?: ( + progress?: firestore.LoadBundleTaskProgress + ) => any; private promiseResolver = new Deferred(); - private promiseFulfilled?: (a: LoadBundleTaskProgress) => any; - private promiseRejected?: (a: Error) => any; + private promiseFulfilled?: ( + progress: firestore.LoadBundleTaskProgress + ) => any; + private promiseRejected?: (err: Error) => any; - private lastProgress: LoadBundleTaskProgress = { + private lastProgress: firestore.LoadBundleTaskProgress = { taskState: 'Running', totalBytes: 0, totalDocuments: 0, @@ -137,9 +123,9 @@ export class LoadBundleTaskImpl implements LoadBundleTask { }; onProgress( - next?: (progress: LoadBundleTaskProgress) => any, + next?: (progress: firestore.LoadBundleTaskProgress) => any, error?: (err: Error) => any, - complete?: (progress?: LoadBundleTaskProgress) => void + complete?: (progress?: firestore.LoadBundleTaskProgress) => void ): Promise { this.progressNext = next; this.progressError = error; @@ -153,15 +139,20 @@ export class LoadBundleTaskImpl implements LoadBundleTask { } then( - onFulfilled?: (a: LoadBundleTaskProgress) => any, + onFulfilled?: (a: firestore.LoadBundleTaskProgress) => any, onRejected?: (a: Error) => any ): Promise { this.promiseFulfilled = onFulfilled; this.promiseRejected = onRejected; return this.promiseResolver.promise; } + /* eslint-enable @typescript-eslint/no-explicit-any */ - completeWith(progress: LoadBundleTaskProgress): void { + /** + * Notifies the completion of loading a bundle, with a provided + * `LoadBundleTaskProgress` object. + */ + completeWith(progress: firestore.LoadBundleTaskProgress): void { let result; if (this.progressComplete) { result = this.progressComplete(progress); @@ -175,9 +166,13 @@ export class LoadBundleTaskImpl implements LoadBundleTask { this.promiseResolver.resolve(result); } + /** + * Notifies a failure of loading a bundle, with a provided `Error` + * as the reason. + */ failedWith(error: Error): void { if (this.progressNext) { - this.lastProgress!.taskState = 'Error'; + this.lastProgress.taskState = 'Error'; this.progressNext(this.lastProgress); } @@ -194,44 +189,65 @@ export class LoadBundleTaskImpl implements LoadBundleTask { this.promiseResolver.reject(result); } - updateProgress(progress: LoadBundleTaskProgress): void { + /** + * Notifies a progress update of loading a bundle. + * @param progress The new progress. + */ + updateProgress(progress: firestore.LoadBundleTaskProgress): void { this.lastProgress = progress; if (this.progressNext) { this.progressNext(progress); } } } -/* eslint-enable @typescript-eslint/no-explicit-any */ export class LoadResult { constructor( - readonly progress: LoadBundleTaskProgress, + readonly progress: firestore.LoadBundleTaskProgress, readonly changedDocs?: MaybeDocumentMap ) {} } +/** + * A class to process the elements from a bundle, load them into local + * storage and provide progress update while loading. + */ export class BundleLoader { - private progress: LoadBundleTaskProgress; + /** The current progress of loading */ + private progress: firestore.LoadBundleTaskProgress; + /** + * The threshold multiplier used to determine whether enough elements are + * batched to be loaded, and a progress update is needed. + */ private step = 0.01; + /** Batched queries to be saved into storage */ private queries: bundleProto.NamedQuery[] = []; + /** Batched documents to be saved into storage */ private documents: BundledDocuments = []; + /** How many bytes in the bundle are being batched. */ private bytesIncrement = 0; + /** How many documents in the bundle are being batched. */ private documentsIncrement = 0; + /** + * A BundleDocumentMetadata is added to the loader, it is saved here while + * we wait for the actual document. + */ private unpairedDocumentMetadata: bundleProto.BundledDocumentMetadata | null = null; constructor( private metadata: bundleProto.BundleMetadata, private localStore: LocalStore ) { - this.progress = { - documentsLoaded: 0, - totalDocuments: metadata.totalDocuments!, - bytesLoaded: 0, - totalBytes: metadata.totalBytes!, - taskState: 'Running' - }; + this.progress = initialProgress('Running', metadata); } + /** + * Adds an element from the bundle to the loader. + * + * If adding this element leads to actually saving the batched elements into + * storage, the returned promise will resolve to a `LoadResult`, otherwise + * it will resolve to null. + */ addSizedElement(element: SizedBundleElement): Promise { debugAssert(!element.isBundleMetadata(), 'Unexpected bundle metadata.'); @@ -293,7 +309,10 @@ export class BundleLoader { return new LoadResult({ ...this.progress }, changedDocs); } - async complete(): Promise { + /** + * Update the progress to 'Success' and return the updated progress. + */ + complete(): firestore.LoadBundleTaskProgress { debugAssert( this.queries.length === 0 && this.documents.length === 0, 'There are more items needs to be saved but complete() is called.' diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index 0723ec9c7bc..7ad50d227ea 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -15,6 +15,7 @@ * limitations under the License. */ +import { LoadBundleTask } from '@firebase/firestore-types'; import { CredentialsProvider } from '../api/credentials'; import { User } from '../auth/user'; import { LocalStore } from '../local/local_store'; @@ -49,7 +50,7 @@ import { MemoryComponentProvider } from './component_provider'; import { BundleReader } from '../util/bundle_reader'; -import { LoadBundleTask, LoadBundleTaskImpl } from './bundle'; +import { LoadBundleTaskImpl } from './bundle'; const LOG_TAG = 'FirestoreClient'; const MAX_CONCURRENT_LIMBO_RESOLUTIONS = 100; diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 420551d65ef..62f110beac1 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -482,10 +482,11 @@ export class SyncEngine implements RemoteSyncer { const result = await loader.addSizedElement(element); if (result) { task.updateProgress(result.progress); - } - if (result && result.changedDocs) { - // eslint-disable-next-line @typescript-eslint/no-floating-promises - this.emitNewSnapsAndNotifyLocalStore(result.changedDocs); + + if (result.changedDocs) { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + this.emitNewSnapsAndNotifyLocalStore(result.changedDocs); + } } element = await reader.nextElement(); @@ -493,7 +494,7 @@ export class SyncEngine implements RemoteSyncer { await this.localStore.saveBundle(metadata); - const completeProgress = await loader.complete(); + const completeProgress = loader.complete(); task.completeWith(completeProgress); } @@ -1329,11 +1330,4 @@ export class MultiTabSyncEngine extends SyncEngine .catch(ignoreIfPrimaryLeaseLoss); } } - - loadBundle( - bundleReader: BundleReader, - task: LoadBundleTaskImpl - ): Promise { - return super.loadBundle(bundleReader, task); - } } diff --git a/packages/firestore/test/integration/api/bundle.test.ts b/packages/firestore/test/integration/api/bundle.test.ts index 10678ec2f00..10f7f562a85 100644 --- a/packages/firestore/test/integration/api/bundle.test.ts +++ b/packages/firestore/test/integration/api/bundle.test.ts @@ -26,17 +26,16 @@ import { import { TestBundleBuilder } from '../../util/bundle_data'; import { DatabaseId } from '../../../src/core/database_info'; import { key } from '../../util/helpers'; -import { LoadBundleTaskProgress } from '../../../src/core/bundle'; import { EventsAccumulator } from '../util/events_accumulator'; -function verifySuccessProgress(p: LoadBundleTaskProgress): void { +function verifySuccessProgress(p: firestore.LoadBundleTaskProgress): void { expect(p.taskState).to.equal('Success'); expect(p.bytesLoaded).to.be.equal(p.totalBytes); expect(p.documentsLoaded).to.equal(p.totalDocuments); } function verifyInProgress( - p: LoadBundleTaskProgress, + p: firestore.LoadBundleTaskProgress, expectedDocuments: number ): void { expect(p.taskState).to.equal('Running'); @@ -83,9 +82,9 @@ apiDescribe('Bundles', (persistence: boolean) => { return withTestDb(persistence, async db => { const builder = bundleWithTestDocs(db); - const progresses: LoadBundleTaskProgress[] = []; - let completeProgress: LoadBundleTaskProgress, - fulfillProgress: LoadBundleTaskProgress; + const progresses: firestore.LoadBundleTaskProgress[] = []; + let completeProgress: firestore.LoadBundleTaskProgress, + fulfillProgress: firestore.LoadBundleTaskProgress; await db .loadBundle( encoder.encode( @@ -132,7 +131,7 @@ apiDescribe('Bundles', (persistence: boolean) => { return withTestDb(persistence, async db => { const builder = bundleWithTestDocs(db); - let fulfillProgress: LoadBundleTaskProgress; + let fulfillProgress: firestore.LoadBundleTaskProgress; await db .loadBundle( encoder.encode( @@ -168,8 +167,8 @@ apiDescribe('Bundles', (persistence: boolean) => { ) ); - let completeProgress: LoadBundleTaskProgress; - const progresses: LoadBundleTaskProgress[] = []; + let completeProgress: firestore.LoadBundleTaskProgress; + const progresses: firestore.LoadBundleTaskProgress[] = []; await db .loadBundle( encoder.encode( diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts index 4c223179067..ff4109d3c5e 100644 --- a/packages/firestore/test/unit/specs/bundle_spec.test.ts +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -241,7 +241,7 @@ describeSpec('Bundles:', [], () => { .watchAcksFull(query, 250) // Backend tells there is no such doc. .expectEvents(query, {}) - // Bundle tells otherwise, leads to limbo resolution. + // Bundle tells otherwise, leads to limbo. .loadBundle(bundleString1) .expectEvents(query, { added: [doc('collection/a', 500, { key: 'b' })], diff --git a/packages/firestore/test/util/bundle_data.ts b/packages/firestore/test/util/bundle_data.ts index 37ea56e9662..125acbb077b 100644 --- a/packages/firestore/test/util/bundle_data.ts +++ b/packages/firestore/test/util/bundle_data.ts @@ -117,7 +117,7 @@ export class TestBundleBuilder { } } -// Setting up test data. +// TODO(wuandy): Ideally, these should use `TestBundleBuilder` above. export const meta: BundleElement = { metadata: { id: 'test-bundle', From 556a0072ffaa8429b1583d7abeb80a017001aba2 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Tue, 9 Jun 2020 11:54:12 -0400 Subject: [PATCH 25/39] Support loading string for real. --- packages/firestore-types/index.d.ts | 2 +- packages/firestore/src/api/database.ts | 2 +- packages/firestore/src/core/firestore_client.ts | 10 ++++++++-- .../test/integration/api/bundle.test.ts | 17 +++++------------ 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/packages/firestore-types/index.d.ts b/packages/firestore-types/index.d.ts index 5e315240c63..7c543b6a9c6 100644 --- a/packages/firestore-types/index.d.ts +++ b/packages/firestore-types/index.d.ts @@ -86,7 +86,7 @@ export class FirebaseFirestore { terminate(): Promise; loadBundle( - bundleData: ArrayBuffer | ReadableStream | string + bundleData: ArrayBuffer | ReadableStream | string ): LoadBundleTask; INTERNAL: { delete: () => Promise }; diff --git a/packages/firestore/src/api/database.ts b/packages/firestore/src/api/database.ts index 06f70ecd7bc..248720ac8f6 100644 --- a/packages/firestore/src/api/database.ts +++ b/packages/firestore/src/api/database.ts @@ -477,7 +477,7 @@ export class Firestore implements firestore.FirebaseFirestore, FirebaseService { } loadBundle( - bundleData: ArrayBuffer | ReadableStream /*| string */ + bundleData: ArrayBuffer | ReadableStream | string ): firestore.LoadBundleTask { this.ensureClientConfigured(); return this._firestoreClient!.loadBundle(bundleData); diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index 7ad50d227ea..c69713925fe 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -526,11 +526,17 @@ export class FirestoreClient { } loadBundle( - data: ReadableStream | ArrayBuffer | Uint8Array + data: ReadableStream | ArrayBuffer | string ): LoadBundleTask { this.verifyNotTerminated(); - const reader = new BundleReader(data); + let content: ReadableStream | ArrayBuffer; + if (typeof data === 'string') { + content = new TextEncoder().encode(data); + } else { + content = data; + } + const reader = new BundleReader(content); const task = new LoadBundleTaskImpl(); this.asyncQueue.enqueueAndForget(() => { return this.syncEngine.loadBundle(reader, task); diff --git a/packages/firestore/test/integration/api/bundle.test.ts b/packages/firestore/test/integration/api/bundle.test.ts index 10f7f562a85..2d78915be2a 100644 --- a/packages/firestore/test/integration/api/bundle.test.ts +++ b/packages/firestore/test/integration/api/bundle.test.ts @@ -87,9 +87,7 @@ apiDescribe('Bundles', (persistence: boolean) => { fulfillProgress: firestore.LoadBundleTaskProgress; await db .loadBundle( - encoder.encode( - builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) - ) + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ) .onProgress( progress => { @@ -134,9 +132,7 @@ apiDescribe('Bundles', (persistence: boolean) => { let fulfillProgress: firestore.LoadBundleTaskProgress; await db .loadBundle( - encoder.encode( - builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) - ) + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ) .then(progress => { fulfillProgress = progress; @@ -162,9 +158,7 @@ apiDescribe('Bundles', (persistence: boolean) => { const builder = bundleWithTestDocs(db); await db.loadBundle( - encoder.encode( - builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) - ) + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ); let completeProgress: firestore.LoadBundleTaskProgress; @@ -210,6 +204,7 @@ apiDescribe('Bundles', (persistence: boolean) => { const builder = bundleWithTestDocs(db); const progress = await db.loadBundle( + // Testing passing non-string bundles. encoder.encode( builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ) @@ -236,9 +231,7 @@ apiDescribe('Bundles', (persistence: boolean) => { // eslint-disable-next-line @typescript-eslint/no-floating-promises expect( otherDb.loadBundle( - encoder.encode( - builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) - ) + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ) ).to.be.rejectedWith('Tried to deserialize key from different project'); }); From adf1504cbc7abcac86e24eca40a638f5823de49e Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 11 Jun 2020 11:09:34 -0400 Subject: [PATCH 26/39] Makes sure SDK still works after loading bad bundles. --- packages/firestore/src/core/sync_engine.ts | 3 --- .../test/integration/api/bundle.test.ts | 19 ++++++++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 62f110beac1..19add4ca533 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -449,9 +449,6 @@ export class SyncEngine implements RemoteSyncer { return ( this.loadBundleAsync(bundleReader, task) - // TODO(wuandy): Loading a bundle will fail the entire SDK if there is - // an error. We really should have a way to run operations on async queue - // and not failing the rest if there is an error. .catch(reason => { task.failedWith(reason); }) diff --git a/packages/firestore/test/integration/api/bundle.test.ts b/packages/firestore/test/integration/api/bundle.test.ts index 2d78915be2a..c5097fd75dd 100644 --- a/packages/firestore/test/integration/api/bundle.test.ts +++ b/packages/firestore/test/integration/api/bundle.test.ts @@ -226,7 +226,7 @@ apiDescribe('Bundles', (persistence: boolean) => { it('load with documents from other projects fails.', () => { return withTestDb(persistence, async db => { - const builder = bundleWithTestDocs(db); + let builder = bundleWithTestDocs(db); return withAlternateTestDb(persistence, async otherDb => { // eslint-disable-next-line @typescript-eslint/no-floating-promises expect( @@ -234,6 +234,23 @@ apiDescribe('Bundles', (persistence: boolean) => { builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ) ).to.be.rejectedWith('Tried to deserialize key from different project'); + + // Verify otherDb still functions, despite loaded a problematic bundle. + builder = bundleWithTestDocs(otherDb); + const progress = await otherDb.loadBundle( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ); + verifySuccessProgress(progress); + + // Read from cache. These documents do not exist in backend, so they can + // only be read from cache. + const snap = await otherDb + .collection('coll-1') + .get({ source: 'cache' }); + expect(toDataArray(snap)).to.deep.equal([ + { k: 'a', bar: 1 }, + { k: 'b', bar: 2 } + ]); }); }); }); From b364ab0e0ca8274a94269708dbf6e996f429b7c1 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 12 Jun 2020 15:45:54 -0400 Subject: [PATCH 27/39] Better spect test. --- packages/firestore/src/core/sync_engine.ts | 9 +- .../test/unit/specs/bundle_spec.test.ts | 98 ++++++++++--------- 2 files changed, 53 insertions(+), 54 deletions(-) diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 19add4ca533..c0adbf3baa7 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -447,12 +447,9 @@ export class SyncEngine implements RemoteSyncer { ): Promise { this.assertSubscribed('loadBundle()'); - return ( - this.loadBundleAsync(bundleReader, task) - .catch(reason => { - task.failedWith(reason); - }) - ); + return this.loadBundleAsync(bundleReader, task).catch(reason => { + task.failedWith(reason); + }); } private async loadBundleAsync( diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts index ff4109d3c5e..143e9838aab 100644 --- a/packages/firestore/test/unit/specs/bundle_spec.test.ts +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -28,7 +28,6 @@ import { import { DocumentKey } from '../../../src/model/document_key'; import * as api from '../../../src/protos/firestore_proto_api'; import { Value } from '../../../src/protos/firestore_proto_api'; -import { TimerId } from '../../../src/util/async_queue'; interface TestBundleDocument { key: DocumentKey; @@ -286,33 +285,37 @@ describeSpec('Bundles:', [], () => { } ); - specTest('Load and observe from secondary clients.', ['multi-client'], () => { - const query = Query.atPath(path('collection')); - const docA = doc('collection/a', 250, { key: 'a' }); - const bundleString1 = bundleWithDocument({ - key: docA.key, - readTime: 500, - createTime: 250, - updateTime: 500, - content: { key: { stringValue: 'b' } } - }); - - return client(0) - .userListens(query) - .watchAcksFull(query, 250, docA) - .expectEvents(query, { - added: [docA] - }) - .client(1) - .userListens(query) - .expectEvents(query, { - added: [docA] - }) - .loadBundle(bundleString1) - .expectEvents(query, { - modified: [doc('collection/a', 500, { key: 'b' })] + specTest( + 'Load and observe from same secondary client.', + ['multi-client'], + () => { + const query = Query.atPath(path('collection')); + const docA = doc('collection/a', 250, { key: 'a' }); + const bundleString1 = bundleWithDocument({ + key: docA.key, + readTime: 500, + createTime: 250, + updateTime: 500, + content: { key: { stringValue: 'b' } } }); - }); + + return client(0) + .userListens(query) + .watchAcksFull(query, 250, docA) + .expectEvents(query, { + added: [docA] + }) + .client(1) + .userListens(query) + .expectEvents(query, { + added: [docA] + }) + .loadBundle(bundleString1) + .expectEvents(query, { + modified: [doc('collection/a', 500, { key: 'b' })] + }); + } + ); specTest( 'Load from primary client and observe from secondary.', @@ -328,27 +331,26 @@ describeSpec('Bundles:', [], () => { content: { key: { stringValue: 'b' } } }); - return ( - client(0) - .userListens(query) - .watchAcksFull(query, 250, docA) - .expectEvents(query, { - added: [docA] - }) - .client(1) - .stealPrimaryLease() - .expectListen(query, 'resume-token-250') - // Bundle tells otherwise, leads to limbo resolution. - .loadBundle(bundleString1) - .client(0) - .runTimer(TimerId.ClientMetadataRefresh) - // Client 0 recovers from its lease loss and applies the updates from - // client 1 - .expectPrimaryState(false) - .expectEvents(query, { - modified: [doc('collection/a', 500, { key: 'b' })] - }) - ); + return client(0) + .userListens(query) + .watchAcksFull(query, 250, docA) + .expectEvents(query, { + added: [docA] + }) + .client(1) + .userListens(query) + .expectEvents(query, { + added: [docA] + }) + .client(0) + .loadBundle(bundleString1) + .expectEvents(query, { + modified: [doc('collection/a', 500, { key: 'b' })] + }) + .client(1) + .expectEvents(query, { + modified: [doc('collection/a', 500, { key: 'b' })] + }); } ); }); From 17ab921d95ad3780953bb3ac37e84485e17c4c1a Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Mon, 29 Jun 2020 15:13:16 -0400 Subject: [PATCH 28/39] Set default bytesPerRead for browser --- packages/firestore/src/platform/browser/byte_stream_reader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/firestore/src/platform/browser/byte_stream_reader.ts b/packages/firestore/src/platform/browser/byte_stream_reader.ts index 65875d4b225..43212dcd9d0 100644 --- a/packages/firestore/src/platform/browser/byte_stream_reader.ts +++ b/packages/firestore/src/platform/browser/byte_stream_reader.ts @@ -23,7 +23,7 @@ import { toByteStreamReaderHelper } from '../../util/byte_stream'; */ export function toByteStreamReader( source: BundleSource, - bytesPerRead: number + bytesPerRead: number = 10240 ): ReadableStreamReader { if (source instanceof Uint8Array) { return toByteStreamReaderHelper(source, bytesPerRead); From 21d4d7cc0d8ea05f222a3a70c358deff0f49934a Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Mon, 29 Jun 2020 16:24:50 -0400 Subject: [PATCH 29/39] Finally ready for initial review. --- packages/firestore/src/core/bundle.ts | 11 ++++++++--- packages/firestore/src/core/sync_engine.ts | 7 +++++++ .../{api => api_internal}/bundle.test.ts | 2 +- .../firestore/test/unit/specs/bundle_spec.test.ts | 2 +- packages/firestore/test/unit/util/bundle.test.ts | 2 +- .../firestore/test/{ => unit}/util/bundle_data.ts | 14 +++++++------- 6 files changed, 25 insertions(+), 13 deletions(-) rename packages/firestore/test/integration/{api => api_internal}/bundle.test.ts (99%) rename packages/firestore/test/{ => unit}/util/bundle_data.ts (92%) diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index 838a1c9f7fc..f4afc0acfc4 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -245,8 +245,11 @@ export class BundleLoader { /** * The threshold multiplier used to determine whether enough elements are * batched to be loaded, and a progress update is needed. + * + * Applies to either number of documents or bytes, triggers storage update + * when either of them cross the threshold. */ - private step = 0.01; + private thresholdMultiplier = 0.01; /** Batched queries to be saved into storage */ private queries: bundleProto.NamedQuery[] = []; /** Batched documents to be saved into storage */ @@ -314,8 +317,10 @@ export class BundleLoader { private async saveAndReportProgress(): Promise { if ( this.unpairedDocumentMetadata || - (this.documentsIncrement < this.progress.totalDocuments * this.step && - this.bytesIncrement < this.progress.totalBytes * this.step) + (this.documentsIncrement < + this.progress.totalDocuments * this.thresholdMultiplier && + this.bytesIncrement < + this.progress.totalBytes * this.thresholdMultiplier) ) { return null; } diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index b1dda3e88fd..85fbd623fe4 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -1381,6 +1381,13 @@ export function newMultiTabSyncEngine( ); } +/** + * Loads a Firestore bundle into the SDK, the returned promise resolves when + * the bundle finished loading. + * + * @param bundleReader Bundle to load into the SDK. + * @param task LoadBundleTask used to update the loading progress to public API. + */ export function loadBundle( syncEngine: SyncEngine, bundleReader: BundleReader, diff --git a/packages/firestore/test/integration/api/bundle.test.ts b/packages/firestore/test/integration/api_internal/bundle.test.ts similarity index 99% rename from packages/firestore/test/integration/api/bundle.test.ts rename to packages/firestore/test/integration/api_internal/bundle.test.ts index c5097fd75dd..0549a2e05d4 100644 --- a/packages/firestore/test/integration/api/bundle.test.ts +++ b/packages/firestore/test/integration/api_internal/bundle.test.ts @@ -23,10 +23,10 @@ import { withAlternateTestDb, withTestDb } from '../util/helpers'; -import { TestBundleBuilder } from '../../util/bundle_data'; import { DatabaseId } from '../../../src/core/database_info'; import { key } from '../../util/helpers'; import { EventsAccumulator } from '../util/events_accumulator'; +import { TestBundleBuilder } from '../../unit/util/bundle_data'; function verifySuccessProgress(p: firestore.LoadBundleTaskProgress): void { expect(p.taskState).to.equal('Success'); diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts index b1c77ad1b3c..771ded9f889 100644 --- a/packages/firestore/test/unit/specs/bundle_spec.test.ts +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -20,7 +20,7 @@ import { doc, path, TestSnapshotVersion, version } from '../../util/helpers'; import { describeSpec, specTest } from './describe_spec'; import { client, spec } from './spec_builder'; -import { TestBundleBuilder } from '../../util/bundle_data'; +import { TestBundleBuilder } from '../util/bundle_data'; import { JSON_SERIALIZER, TEST_DATABASE_ID diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index 2509131cd32..9d6527b5fc2 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -38,7 +38,7 @@ import { doc2String, doc1, doc2 -} from '../../util/bundle_data'; +} from './bundle_data'; use(chaiAsPromised); diff --git a/packages/firestore/test/util/bundle_data.ts b/packages/firestore/test/unit/util/bundle_data.ts similarity index 92% rename from packages/firestore/test/util/bundle_data.ts rename to packages/firestore/test/unit/util/bundle_data.ts index 4c176ac56eb..3984c2be8d3 100644 --- a/packages/firestore/test/util/bundle_data.ts +++ b/packages/firestore/test/unit/util/bundle_data.ts @@ -17,13 +17,13 @@ import { BundledQuery, BundleElement -} from '../../src/protos/firestore_bundle_proto'; -import { DatabaseId } from '../../src/core/database_info'; -import * as api from '../../src/protos/firestore_proto_api'; -import { Value } from '../../src/protos/firestore_proto_api'; -import { JsonProtoSerializer, toName } from '../../src/remote/serializer'; -import { DocumentKey } from '../../src/model/document_key'; -import { newSerializer } from '../../src/platform/serializer'; +} from '../../../src/protos/firestore_bundle_proto'; +import { DatabaseId } from '../../../src/core/database_info'; +import * as api from '../../../src/protos/firestore_proto_api'; +import { Value } from '../../../src/protos/firestore_proto_api'; +import { JsonProtoSerializer, toName } from '../../../src/remote/serializer'; +import { DocumentKey } from '../../../src/model/document_key'; +import { newSerializer } from '../../../src/platform/serializer'; function lengthPrefixedString(o: {}): string { const str = JSON.stringify(o); From bf085ce02ed9c404134aa055ef5f0dd98cfa13a2 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Wed, 1 Jul 2020 18:04:39 -0400 Subject: [PATCH 30/39] Address comments batch 1 --- packages/firestore-types/index.d.ts | 4 +- packages/firestore/src/api/bundle.ts | 108 ++++++++++++ packages/firestore/src/core/bundle.ts | 162 +++++------------- .../firestore/src/core/firestore_client.ts | 8 +- packages/firestore/src/core/sync_engine.ts | 44 ++--- packages/firestore/src/core/view.ts | 6 +- .../platform/browser/byte_stream_reader.ts | 3 +- .../src/platform/byte_stream_reader.ts | 9 +- .../integration/api_internal/bundle.test.ts | 135 +++++++-------- .../test/unit/specs/bundle_spec.test.ts | 96 +++++------ .../test/unit/specs/spec_test_runner.ts | 10 +- 11 files changed, 298 insertions(+), 287 deletions(-) create mode 100644 packages/firestore/src/api/bundle.ts diff --git a/packages/firestore-types/index.d.ts b/packages/firestore-types/index.d.ts index 17c484b68ea..404ce935912 100644 --- a/packages/firestore-types/index.d.ts +++ b/packages/firestore-types/index.d.ts @@ -104,8 +104,8 @@ export interface LoadBundleTask { onProgress( next?: (progress: LoadBundleTaskProgress) => any, error?: (error: Error) => any, - complete?: (progress?: LoadBundleTaskProgress) => any - ): Promise; + complete?: () => void + ): Promise; then( onFulfilled?: (a: LoadBundleTaskProgress) => any, diff --git a/packages/firestore/src/api/bundle.ts b/packages/firestore/src/api/bundle.ts new file mode 100644 index 00000000000..c02f267d4ca --- /dev/null +++ b/packages/firestore/src/api/bundle.ts @@ -0,0 +1,108 @@ +/** + * @license + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as firestore from '@firebase/firestore-types'; +import { Deferred } from '../util/promise'; + +export class LoadBundleTask implements firestore.LoadBundleTask { + private _progressResolver = new Deferred(); + private _userProgressHandler?: ( + progress: firestore.LoadBundleTaskProgress + ) => unknown; + private _userProgressErrorHandler?: (err: Error) => unknown; + private _userProgressCompleteHandler?: () => void; + + private _promiseResolver = new Deferred(); + + private _lastProgress: firestore.LoadBundleTaskProgress = { + taskState: 'Running', + totalBytes: 0, + totalDocuments: 0, + bytesLoaded: 0, + documentsLoaded: 0 + }; + + onProgress( + next?: (progress: firestore.LoadBundleTaskProgress) => unknown, + error?: (err: Error) => unknown, + complete?: () => void + ): Promise { + this._userProgressHandler = next; + this._userProgressErrorHandler = error; + this._userProgressCompleteHandler = complete; + return this._progressResolver.promise; + } + + catch(onRejected: (a: Error) => unknown): Promise { + return this._promiseResolver.promise.catch(onRejected); + } + + then( + onFulfilled?: (a: firestore.LoadBundleTaskProgress) => unknown, + onRejected?: (a: Error) => unknown + ): Promise { + return this._promiseResolver.promise.then(onFulfilled, onRejected); + } + + /** + * Notifies the completion of loading a bundle, with a provided + * `LoadBundleTaskProgress` object. + */ + _completeWith(progress: firestore.LoadBundleTaskProgress): void { + this._updateProgress(progress); + if (this._userProgressCompleteHandler) { + this._userProgressCompleteHandler(); + } + this._progressResolver.resolve(); + + this._promiseResolver.resolve(progress); + } + + /** + * Notifies a failure of loading a bundle, with a provided `Error` + * as the reason. + */ + _failedWith(error: Error): void { + this._lastProgress.taskState = 'Error'; + + if (this._userProgressHandler) { + this._userProgressHandler(this._lastProgress); + } + + if (this._userProgressErrorHandler) { + this._userProgressErrorHandler(error); + } + this._progressResolver.reject(error); + + this._promiseResolver.reject(error); + } + + /** + * Notifies a progress update of loading a bundle. + * @param progress The new progress. + */ + _updateProgress(progress: firestore.LoadBundleTaskProgress): void { + if (this._lastProgress.taskState === 'Error') { + return; + } + + this._lastProgress = progress; + if (this._userProgressHandler) { + this._userProgressHandler(progress); + } + } +} diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index f4afc0acfc4..ee35a7d57df 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -36,7 +36,6 @@ import { } from '../local/local_store'; import { SizedBundleElement } from '../util/bundle_reader'; import { MaybeDocumentMap } from '../model/collections'; -import { Deferred } from '../util/promise'; import { BundleMetadata } from '../protos/firestore_bundle_proto'; /** @@ -110,122 +109,35 @@ export class BundleConverter { } /** - * Returns a `LoadBundleTaskProgress` representing the first progress of + * Returns a `LoadBundleTaskProgress` representing the initial progress of * loading a bundle. */ export function initialProgress( - state: firestore.TaskState, metadata: BundleMetadata ): firestore.LoadBundleTaskProgress { return { - taskState: state, - documentsLoaded: state === 'Success' ? metadata.totalDocuments! : 0, - bytesLoaded: state === 'Success' ? metadata.totalBytes! : 0, + taskState: 'Running', + documentsLoaded: 0, + bytesLoaded: 0, totalDocuments: metadata.totalDocuments!, totalBytes: metadata.totalBytes! }; } -/* eslint-disable @typescript-eslint/no-explicit-any */ -export class LoadBundleTaskImpl implements firestore.LoadBundleTask { - private progressResolver = new Deferred(); - private progressNext?: (progress: firestore.LoadBundleTaskProgress) => any; - private progressError?: (err: Error) => any; - private progressComplete?: ( - progress?: firestore.LoadBundleTaskProgress - ) => any; - - private promiseResolver = new Deferred(); - private promiseFulfilled?: ( - progress: firestore.LoadBundleTaskProgress - ) => any; - private promiseRejected?: (err: Error) => any; - - private lastProgress: firestore.LoadBundleTaskProgress = { - taskState: 'Running', - totalBytes: 0, - totalDocuments: 0, - bytesLoaded: 0, - documentsLoaded: 0 +/** + * Returns a `LoadBundleTaskProgress` representing the progress if the bundle + * is already loaded, and we are skipping current loading. + */ +export function skipLoadingProgress( + metadata: BundleMetadata +): firestore.LoadBundleTaskProgress { + return { + taskState: 'Success', + documentsLoaded: metadata.totalDocuments!, + bytesLoaded: metadata.totalBytes!, + totalDocuments: metadata.totalDocuments!, + totalBytes: metadata.totalBytes! }; - - onProgress( - next?: (progress: firestore.LoadBundleTaskProgress) => any, - error?: (err: Error) => any, - complete?: (progress?: firestore.LoadBundleTaskProgress) => void - ): Promise { - this.progressNext = next; - this.progressError = error; - this.progressComplete = complete; - return this.progressResolver.promise; - } - - catch(onRejected: (a: Error) => any): Promise { - this.promiseRejected = onRejected; - return this.promiseResolver.promise; - } - - then( - onFulfilled?: (a: firestore.LoadBundleTaskProgress) => any, - onRejected?: (a: Error) => any - ): Promise { - this.promiseFulfilled = onFulfilled; - this.promiseRejected = onRejected; - return this.promiseResolver.promise; - } - /* eslint-enable @typescript-eslint/no-explicit-any */ - - /** - * Notifies the completion of loading a bundle, with a provided - * `LoadBundleTaskProgress` object. - */ - completeWith(progress: firestore.LoadBundleTaskProgress): void { - let result; - if (this.progressComplete) { - result = this.progressComplete(progress); - } - this.progressResolver.resolve(result); - - result = undefined; - if (this.promiseFulfilled) { - result = this.promiseFulfilled(progress); - } - this.promiseResolver.resolve(result); - } - - /** - * Notifies a failure of loading a bundle, with a provided `Error` - * as the reason. - */ - failedWith(error: Error): void { - if (this.progressNext) { - this.lastProgress.taskState = 'Error'; - this.progressNext(this.lastProgress); - } - - let result; - if (this.progressError) { - result = this.progressError(error); - } - this.progressResolver.reject(result); - - result = undefined; - if (this.promiseRejected) { - this.promiseRejected(error); - } - this.promiseResolver.reject(result); - } - - /** - * Notifies a progress update of loading a bundle. - * @param progress The new progress. - */ - updateProgress(progress: firestore.LoadBundleTaskProgress): void { - this.lastProgress = progress; - if (this.progressNext) { - this.progressNext(progress); - } - } } export class LoadResult { @@ -246,18 +158,24 @@ export class BundleLoader { * The threshold multiplier used to determine whether enough elements are * batched to be loaded, and a progress update is needed. * - * Applies to either number of documents or bytes, triggers storage update - * when either of them cross the threshold. + * Applies to both `documentsBuffered` and `bytesBuffered`, triggers storage + * update and reports progress when either of them cross the threshold. */ private thresholdMultiplier = 0.01; /** Batched queries to be saved into storage */ private queries: bundleProto.NamedQuery[] = []; /** Batched documents to be saved into storage */ private documents: BundledDocuments = []; - /** How many bytes in the bundle are being batched. */ - private bytesIncrement = 0; - /** How many documents in the bundle are being batched. */ - private documentsIncrement = 0; + /** + * How many bytes from the bundle are being buffered since last progress + * update. + */ + private bytesBuffered = 0; + /** + * How many documents from the bundle are being buffered since last progress + * update. + */ + private documentsBuffered = 0; /** * A BundleDocumentMetadata is added to the loader, it is saved here while * we wait for the actual document. @@ -268,7 +186,7 @@ export class BundleLoader { private metadata: bundleProto.BundleMetadata, private localStore: LocalStore ) { - this.progress = initialProgress('Running', metadata); + this.progress = initialProgress(metadata); } /** @@ -281,7 +199,7 @@ export class BundleLoader { addSizedElement(element: SizedBundleElement): Promise { debugAssert(!element.isBundleMetadata(), 'Unexpected bundle metadata.'); - this.bytesIncrement += element.byteLength; + this.bytesBuffered += element.byteLength; if (element.payload.namedQuery) { this.queries.push(element.payload.namedQuery); } @@ -294,7 +212,7 @@ export class BundleLoader { metadata: element.payload.documentMetadata, document: undefined }); - this.documentsIncrement += 1; + this.documentsBuffered += 1; } } @@ -307,7 +225,7 @@ export class BundleLoader { metadata: this.unpairedDocumentMetadata!, document: element.payload.document }); - this.documentsIncrement += 1; + this.documentsBuffered += 1; this.unpairedDocumentMetadata = null; } @@ -317,9 +235,9 @@ export class BundleLoader { private async saveAndReportProgress(): Promise { if ( this.unpairedDocumentMetadata || - (this.documentsIncrement < + (this.documentsBuffered < this.progress.totalDocuments * this.thresholdMultiplier && - this.bytesIncrement < + this.bytesBuffered < this.progress.totalBytes * this.thresholdMultiplier) ) { return null; @@ -334,10 +252,10 @@ export class BundleLoader { changedDocs = await applyBundleDocuments(this.localStore, this.documents); } - this.progress.bytesLoaded += this.bytesIncrement; - this.progress.documentsLoaded += this.documentsIncrement; - this.bytesIncrement = 0; - this.documentsIncrement = 0; + this.progress.bytesLoaded += this.bytesBuffered; + this.progress.documentsLoaded += this.documentsBuffered; + this.bytesBuffered = 0; + this.documentsBuffered = 0; this.queries = []; this.documents = []; @@ -350,7 +268,7 @@ export class BundleLoader { complete(): firestore.LoadBundleTaskProgress { debugAssert( this.queries.length === 0 && this.documents.length === 0, - 'There are more items needs to be saved but complete() is called.' + 'There are more items needs to be saved but complete() was called.' ); this.progress.taskState = 'Success'; diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index f81b72c34b4..0c42d132d1b 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import { LoadBundleTask } from '@firebase/firestore-types'; +import * as firestore from '@firebase/firestore-types'; import { CredentialsProvider } from '../api/credentials'; import { User } from '../auth/user'; import { LocalStore } from '../local/local_store'; @@ -49,7 +49,7 @@ import { MemoryComponentProvider } from './component_provider'; import { BundleReader } from '../util/bundle_reader'; -import { LoadBundleTaskImpl } from './bundle'; +import { LoadBundleTask } from '../api/bundle'; import { newConnection } from '../platform/connection'; import { newSerializer } from '../platform/serializer'; import { toByteStreamReader } from '../platform/byte_stream_reader'; @@ -519,7 +519,7 @@ export class FirestoreClient { loadBundle( data: ReadableStream | ArrayBuffer | string - ): LoadBundleTask { + ): firestore.LoadBundleTask { this.verifyNotTerminated(); let content: ReadableStream | ArrayBuffer; @@ -529,7 +529,7 @@ export class FirestoreClient { content = data; } const reader = new BundleReader(toByteStreamReader(content)); - const task = new LoadBundleTaskImpl(); + const task = new LoadBundleTask(); this.asyncQueue.enqueueAndForget(() => { return loadBundle(this.syncEngine, reader, task); }); diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 85fbd623fe4..7f4b1836ccb 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -77,8 +77,9 @@ import { ViewSnapshot } from './view_snapshot'; import { AsyncQueue, wrapInUserErrorIfRecoverable } from '../util/async_queue'; import { TransactionRunner } from './transaction_runner'; import { BundleReader } from '../util/bundle_reader'; -import { BundleLoader, initialProgress, LoadBundleTaskImpl } from './bundle'; +import { BundleLoader, initialProgress, skipLoadingProgress } from './bundle'; import { Datastore } from '../remote/datastore'; +import { LoadBundleTask } from '../api/bundle'; const LOG_TAG = 'SyncEngine'; @@ -304,11 +305,6 @@ class SyncEngineImpl implements SyncEngine { this.syncEngineListener = syncEngineListener; } - /** - * Initiates the new listen, resolves promise when listen enqueued to the - * server. All the subsequent view snapshots or errors are sent to the - * subscribed handlers. Returns the initial snapshot. - */ async listen(query: Query): Promise { this.assertSubscribed('listen()'); @@ -848,7 +844,8 @@ class SyncEngineImpl implements SyncEngine { async emitNewSnapsAndNotifyLocalStore( changes: MaybeDocumentMap, - remoteEvent?: RemoteEvent + remoteEvent?: RemoteEvent, + fromBundle: boolean = false ): Promise { const newSnaps: ViewSnapshot[] = []; const docChangesInAllViews: LocalViewChanges[] = []; @@ -880,7 +877,8 @@ class SyncEngineImpl implements SyncEngine { const viewChange = queryView.view.applyChanges( viewDocChanges, /* updateLimboDocuments= */ this.isPrimaryClient, - targetChange + targetChange, + fromBundle ); this.updateTrackedLimbos( queryView.targetId, @@ -1388,33 +1386,35 @@ export function newMultiTabSyncEngine( * @param bundleReader Bundle to load into the SDK. * @param task LoadBundleTask used to update the loading progress to public API. */ -export function loadBundle( +export async function loadBundle( syncEngine: SyncEngine, bundleReader: BundleReader, - task: LoadBundleTaskImpl + task: LoadBundleTask ): Promise { const syncEngineImpl = debugCast(syncEngine, SyncEngineImpl); syncEngineImpl.assertSubscribed('loadBundle()'); - return loadBundleAsync(syncEngineImpl, bundleReader, task).catch(reason => { - task.failedWith(reason); - }); + try { + await loadBundleImpl(syncEngineImpl, bundleReader, task); + } catch (e) { + task._failedWith(e); + } } -async function loadBundleAsync( +async function loadBundleImpl( syncEngine: SyncEngineImpl, reader: BundleReader, - task: LoadBundleTaskImpl + task: LoadBundleTask ): Promise { const metadata = await reader.getMetadata(); const skip = await hasNewerBundle(syncEngine.localStore, metadata); if (skip) { await reader.close(); - task.completeWith(initialProgress('Success', metadata)); + task._completeWith(skipLoadingProgress(metadata)); return; } - task.updateProgress(initialProgress('Running', metadata)); + task._updateProgress(initialProgress(metadata)); const loader = new BundleLoader(metadata, syncEngine.localStore); let element = await reader.nextElement(); @@ -1425,11 +1425,15 @@ async function loadBundleAsync( ); const result = await loader.addSizedElement(element); if (result) { - task.updateProgress(result.progress); + task._updateProgress(result.progress); if (result.changedDocs) { // eslint-disable-next-line @typescript-eslint/no-floating-promises - syncEngine.emitNewSnapsAndNotifyLocalStore(result.changedDocs); + syncEngine.emitNewSnapsAndNotifyLocalStore( + result.changedDocs, + /* remoteEvent */ undefined, + /* fromBundle */ true + ); } } @@ -1439,5 +1443,5 @@ async function loadBundleAsync( await saveBundle(syncEngine.localStore, metadata); const completeProgress = loader.complete(); - task.completeWith(completeProgress); + task._completeWith(completeProgress); } diff --git a/packages/firestore/src/core/view.ts b/packages/firestore/src/core/view.ts index c9f34f08103..1cec1fcb522 100644 --- a/packages/firestore/src/core/view.ts +++ b/packages/firestore/src/core/view.ts @@ -280,13 +280,15 @@ export class View { * change. * @param targetChange A target change to apply for computing limbo docs and * sync state. + * @param fromBundle Whether the changes are from applying a bundle file. * @return A new ViewChange with the given docs, changes, and sync state. */ // PORTING NOTE: The iOS/Android clients always compute limbo document changes. applyChanges( docChanges: ViewDocumentChanges, updateLimboDocuments: boolean, - targetChange?: TargetChange + targetChange?: TargetChange, + fromBundle: boolean = false ): ViewChange { debugAssert( !docChanges.needsRefill, @@ -323,7 +325,7 @@ export class View { oldDocs, changes, docChanges.mutatedKeys, - newSyncState === SyncState.Local, + newSyncState === SyncState.Local || fromBundle, syncStateChanged, /* excludesMetadataChanges= */ false ); diff --git a/packages/firestore/src/platform/browser/byte_stream_reader.ts b/packages/firestore/src/platform/browser/byte_stream_reader.ts index 43212dcd9d0..0f5a64c6d23 100644 --- a/packages/firestore/src/platform/browser/byte_stream_reader.ts +++ b/packages/firestore/src/platform/browser/byte_stream_reader.ts @@ -17,13 +17,14 @@ import { BundleSource } from '../../util/bundle_reader'; import { toByteStreamReaderHelper } from '../../util/byte_stream'; +import { DEFAULT_BYTES_PER_READ } from '../byte_stream_reader'; /** * On web, a `ReadableStream` is wrapped around by a `ByteStreamReader`. */ export function toByteStreamReader( source: BundleSource, - bytesPerRead: number = 10240 + bytesPerRead: number = DEFAULT_BYTES_PER_READ ): ReadableStreamReader { if (source instanceof Uint8Array) { return toByteStreamReaderHelper(source, bytesPerRead); diff --git a/packages/firestore/src/platform/byte_stream_reader.ts b/packages/firestore/src/platform/byte_stream_reader.ts index 689159d845d..2f69db539b8 100644 --- a/packages/firestore/src/platform/byte_stream_reader.ts +++ b/packages/firestore/src/platform/byte_stream_reader.ts @@ -21,9 +21,16 @@ import * as rn from './rn/byte_stream_reader'; import * as browser from './browser/byte_stream_reader'; import { isNode, isReactNative } from '@firebase/util'; +/** + * For the byte streams where we have control (like backed by a UInt8Array), + * how many bytes to read each time when `ReadableStreamReader.read()` is + * called. + */ +export const DEFAULT_BYTES_PER_READ = 10240; + export function toByteStreamReader( source: BundleSource, - bytesPerRead: number = 10240 + bytesPerRead: number = DEFAULT_BYTES_PER_READ ): ReadableStreamReader { if (isNode()) { return node.toByteStreamReader(source, bytesPerRead); diff --git a/packages/firestore/test/integration/api_internal/bundle.test.ts b/packages/firestore/test/integration/api_internal/bundle.test.ts index 0549a2e05d4..0cc51c4dad1 100644 --- a/packages/firestore/test/integration/api_internal/bundle.test.ts +++ b/packages/firestore/test/integration/api_internal/bundle.test.ts @@ -39,8 +39,8 @@ function verifyInProgress( expectedDocuments: number ): void { expect(p.taskState).to.equal('Running'); - expect(p.bytesLoaded).lte(p.totalBytes); - expect(p.documentsLoaded).lte(p.totalDocuments); + expect(p.bytesLoaded <= p.totalBytes).to.be.true; + expect(p.documentsLoaded <= p.totalDocuments).to.be.true; expect(p.documentsLoaded).to.equal(expectedDocuments); } @@ -78,82 +78,69 @@ apiDescribe('Bundles', (persistence: boolean) => { return builder; } - it('load with documents only with on progress and promise interface.', () => { + function verifySnapEqualTestDocs(snap: firestore.QuerySnapshot): void { + expect(toDataArray(snap)).to.deep.equal([ + { k: 'a', bar: 1 }, + { k: 'b', bar: 2 } + ]); + } + + it('load with documents only with on progress and promise interface', () => { return withTestDb(persistence, async db => { const builder = bundleWithTestDocs(db); - const progresses: firestore.LoadBundleTaskProgress[] = []; - let completeProgress: firestore.LoadBundleTaskProgress, - fulfillProgress: firestore.LoadBundleTaskProgress; - await db - .loadBundle( - builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) - ) - .onProgress( - progress => { - progresses.push(progress); - }, - err => { - throw err; - }, - progress => { - completeProgress = progress!; - return progress; - } - ) - .then(progress => { - fulfillProgress = progress; - }) - .catch(err => { - throw err; - }); + const progressEvents: firestore.LoadBundleTaskProgress[] = []; + let completeCalled = false; + const task = db.loadBundle( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ); + await task.onProgress( + progress => { + progressEvents.push(progress); + }, + undefined, + () => { + completeCalled = true; + } + ); + let fulfillProgress: firestore.LoadBundleTaskProgress; + await task.then(progress => { + fulfillProgress = progress; + }); - verifySuccessProgress(completeProgress!); - verifySuccessProgress(fulfillProgress!); - expect(progresses.length).to.equal(3); - verifyInProgress(progresses[0], 0); - verifyInProgress(progresses[1], 1); - verifyInProgress(progresses[2], 2); + expect(completeCalled).to.be.true; + expect(progressEvents.length).to.equal(4); + verifyInProgress(progressEvents[0], 0); + verifyInProgress(progressEvents[1], 1); + verifyInProgress(progressEvents[2], 2); + verifySuccessProgress(progressEvents[3]); + expect(fulfillProgress!).to.deep.equal(progressEvents[3]); // Read from cache. These documents do not exist in backend, so they can // only be read from cache. const snap = await db.collection('coll-1').get({ source: 'cache' }); - expect(toDataArray(snap)).to.deep.equal([ - { k: 'a', bar: 1 }, - { k: 'b', bar: 2 } - ]); + verifySnapEqualTestDocs(snap); }); }); - it('load with documents with promise interface.', () => { + it('load with documents with promise interface', () => { return withTestDb(persistence, async db => { const builder = bundleWithTestDocs(db); - let fulfillProgress: firestore.LoadBundleTaskProgress; - await db - .loadBundle( - builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) - ) - .then(progress => { - fulfillProgress = progress; - }) - .catch(err => { - throw err; - }); + const fulfillProgress: firestore.LoadBundleTaskProgress = await db.loadBundle( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) + ); verifySuccessProgress(fulfillProgress!); // Read from cache. These documents do not exist in backend, so they can // only be read from cache. const snap = await db.collection('coll-1').get({ source: 'cache' }); - expect(toDataArray(snap)).to.deep.equal([ - { k: 'a', bar: 1 }, - { k: 'b', bar: 2 } - ]); + verifySnapEqualTestDocs(snap); }); }); - it('load for a second time skips.', () => { + it('load for a second time skips', () => { return withTestDb(persistence, async db => { const builder = bundleWithTestDocs(db); @@ -161,8 +148,8 @@ apiDescribe('Bundles', (persistence: boolean) => { builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ); - let completeProgress: firestore.LoadBundleTaskProgress; - const progresses: firestore.LoadBundleTaskProgress[] = []; + let completeCalled = false; + const progressEvents: firestore.LoadBundleTaskProgress[] = []; await db .loadBundle( encoder.encode( @@ -171,29 +158,28 @@ apiDescribe('Bundles', (persistence: boolean) => { ) .onProgress( progress => { - progresses.push(progress); + progressEvents.push(progress); }, error => {}, - progress => { - completeProgress = progress!; + () => { + completeCalled = true; } ); - // No loading actually happened in the second `loadBundle` call. - expect(progresses).to.be.empty; - verifySuccessProgress(completeProgress!); + expect(completeCalled).to.be.true; + // No loading actually happened in the second `loadBundle` call only the + // success progress is recorded. + expect(progressEvents.length).to.equal(1); + verifySuccessProgress(progressEvents[0]); // Read from cache. These documents do not exist in backend, so they can // only be read from cache. const snap = await db.collection('coll-1').get({ source: 'cache' }); - expect(toDataArray(snap)).to.deep.equal([ - { k: 'a', bar: 1 }, - { k: 'b', bar: 2 } - ]); + verifySnapEqualTestDocs(snap); }); }); - it('load with documents already pulled from backend.', () => { + it('load with documents already pulled from backend', () => { return withTestDb(persistence, async db => { await db.doc('coll-1/a').set({ k: 'a', bar: 0 }); await db.doc('coll-1/b').set({ k: 'b', bar: 0 }); @@ -204,7 +190,7 @@ apiDescribe('Bundles', (persistence: boolean) => { const builder = bundleWithTestDocs(db); const progress = await db.loadBundle( - // Testing passing non-string bundles. + // Testing passing in non-string bundles. encoder.encode( builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ) @@ -224,7 +210,7 @@ apiDescribe('Bundles', (persistence: boolean) => { }); }); - it('load with documents from other projects fails.', () => { + it('load with documents from other projects fails', () => { return withTestDb(persistence, async db => { let builder = bundleWithTestDocs(db); return withAlternateTestDb(persistence, async otherDb => { @@ -237,20 +223,17 @@ apiDescribe('Bundles', (persistence: boolean) => { // Verify otherDb still functions, despite loaded a problematic bundle. builder = bundleWithTestDocs(otherDb); - const progress = await otherDb.loadBundle( + const finalProgress = await otherDb.loadBundle( builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ); - verifySuccessProgress(progress); + verifySuccessProgress(finalProgress); // Read from cache. These documents do not exist in backend, so they can // only be read from cache. const snap = await otherDb .collection('coll-1') .get({ source: 'cache' }); - expect(toDataArray(snap)).to.deep.equal([ - { k: 'a', bar: 1 }, - { k: 'b', bar: 2 } - ]); + verifySnapEqualTestDocs(snap); }); }); }); diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts index 771ded9f889..f1ad153488d 100644 --- a/packages/firestore/test/unit/specs/bundle_spec.test.ts +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -58,8 +58,8 @@ function bundleWithDocument(testDoc: TestBundleDocument): string { ); } -describeSpec('Bundles:', [], () => { - specTest('Newer docs from bundles should overwrite cache.', [], () => { +describeSpec('Bundles:', ['no-ios', 'no-android'], () => { + specTest('Newer docs from bundles should overwrite cache', [], () => { const query1 = Query.atPath(path('collection')); const docA = doc('collection/a', 1000, { key: 'a' }); const docAChanged = doc('collection/a', 2999, { key: 'b' }); @@ -77,10 +77,10 @@ describeSpec('Bundles:', [], () => { .watchAcksFull(query1, 1000, docA) .expectEvents(query1, { added: [docA] }) .loadBundle(bundleString) - .expectEvents(query1, { modified: [docAChanged] }); + .expectEvents(query1, { modified: [docAChanged], fromCache: true }); }); - specTest('Newer deleted docs from bundles should delete cache.', [], () => { + specTest('Newer deleted docs from bundles should delete cache', [], () => { const query1 = Query.atPath(path('collection')); const docA = doc('collection/a', 1000, { key: 'a' }); @@ -94,10 +94,10 @@ describeSpec('Bundles:', [], () => { .watchAcksFull(query1, 1000, docA) .expectEvents(query1, { added: [docA] }) .loadBundle(bundleString) - .expectEvents(query1, { removed: [docA] }); + .expectEvents(query1, { removed: [docA], fromCache: true }); }); - specTest('Older deleted docs from bundles should do nothing.', [], () => { + specTest('Older deleted docs from bundles should do nothing', [], () => { const query1 = Query.atPath(path('collection')); const docA = doc('collection/a', 1000, { key: 'a' }); @@ -117,13 +117,13 @@ describeSpec('Bundles:', [], () => { }); specTest( - 'Newer docs from bundles should raise snapshot only when watch catches up with acknowledged writes.', + 'Newer docs from bundles should raise snapshot only when watch catches up with acknowledged writes', [], () => { const query = Query.atPath(path('collection')); const docA = doc('collection/a', 250, { key: 'a' }); - const bundleString1 = bundleWithDocument({ + const bundleBeforeMutationAck = bundleWithDocument({ key: docA.key, readTime: 500, createTime: 250, @@ -131,7 +131,7 @@ describeSpec('Bundles:', [], () => { content: { key: { stringValue: 'b' } } }); - const bundleString2 = bundleWithDocument({ + const bundleAfterMutationAck = bundleWithDocument({ key: docA.key, readTime: 1001, createTime: 250, @@ -159,35 +159,28 @@ describeSpec('Bundles:', [], () => { hasPendingWrites: true }) .writeAcks('collection/a', 1000) - // loading bundleString1 will not raise snapshots, because it is before + // loading bundleBeforeMutationAck will not raise snapshots, because it is before // the acknowledged mutation. - .loadBundle(bundleString1) - // loading bundleString2 will raise a snapshot, because it is after + .loadBundle(bundleBeforeMutationAck) + // loading bundleAfterMutationAck will raise a snapshot, because it is after // the acknowledged mutation. - .loadBundle(bundleString2) + .loadBundle(bundleAfterMutationAck) .expectEvents(query, { - modified: [doc('collection/a', 1001, { key: 'fromBundle' })] + modified: [doc('collection/a', 1001, { key: 'fromBundle' })], + fromCache: true }) ); } ); specTest( - 'Newer docs from bundles should keep not raise snapshot if there are unacknowledged writes.', + 'Newer docs from bundles should keep not raise snapshot if there are unacknowledged writes', [], () => { const query = Query.atPath(path('collection')); const docA = doc('collection/a', 250, { key: 'a' }); - const bundleString1 = bundleWithDocument({ - key: docA.key, - readTime: 500, - createTime: 250, - updateTime: 500, - content: { key: { stringValue: 'b' } } - }); - - const bundleString2 = bundleWithDocument({ + const bundleString = bundleWithDocument({ key: docA.key, readTime: 1001, createTime: 250, @@ -215,15 +208,14 @@ describeSpec('Bundles:', [], () => { ], hasPendingWrites: true }) - // Loading both bundles will not raise snapshots, because of the + // Loading the bundle will not raise snapshots, because the // mutation is not acknowledged. - .loadBundle(bundleString1) - .loadBundle(bundleString2) + .loadBundle(bundleString) ); } ); - specTest('Newer docs from bundles might lead to limbo doc.', [], () => { + specTest('Newer docs from bundles might lead to limbo doc', [], () => { const query = Query.atPath(path('collection')); const docA = doc('collection/a', 1000, { key: 'a' }); const bundleString1 = bundleWithDocument({ @@ -252,7 +244,7 @@ describeSpec('Bundles:', [], () => { }); specTest( - 'Load from secondary clients and observe from primary.', + 'Load from secondary clients and observe from primary', ['multi-client'], () => { const query = Query.atPath(path('collection')); @@ -265,29 +257,26 @@ describeSpec('Bundles:', [], () => { content: { key: { stringValue: 'b' } } }); - return ( - client(0) - .userListens(query) - .watchAcksFull(query, 250, docA) - .expectEvents(query, { - added: [docA] - }) - .client(1) - // Bundle tells otherwise, leads to limbo resolution. - .loadBundle(bundleString1) - .client(0) - .becomeVisible() - // TODO(wuandy): Loading from secondary client does not notify other - // clients for now. We need to fix it and uncomment below. - // .expectEvents(query, { - // modified: [doc('collection/a', 500, { key: 'b' })], - // }) - ); + return client(0) + .userListens(query) + .watchAcksFull(query, 250, docA) + .expectEvents(query, { + added: [docA] + }) + .client(1) + .loadBundle(bundleString1) + .client(0) + .becomeVisible(); + // TODO(wuandy): Loading from secondary client does not notify other + // clients for now. We need to fix it and uncomment below. + // .expectEvents(query, { + // modified: [doc('collection/a', 500, { key: 'b' })], + // }) } ); specTest( - 'Load and observe from same secondary client.', + 'Load and observe from same secondary client', ['multi-client'], () => { const query = Query.atPath(path('collection')); @@ -313,13 +302,14 @@ describeSpec('Bundles:', [], () => { }) .loadBundle(bundleString1) .expectEvents(query, { - modified: [doc('collection/a', 500, { key: 'b' })] + modified: [doc('collection/a', 500, { key: 'b' })], + fromCache: true }); } ); specTest( - 'Load from primary client and observe from secondary.', + 'Load from primary client and observe from secondary', ['multi-client'], () => { const query = Query.atPath(path('collection')); @@ -346,11 +336,13 @@ describeSpec('Bundles:', [], () => { .client(0) .loadBundle(bundleString1) .expectEvents(query, { - modified: [doc('collection/a', 500, { key: 'b' })] + modified: [doc('collection/a', 500, { key: 'b' })], + fromCache: true }) .client(1) .expectEvents(query, { - modified: [doc('collection/a', 500, { key: 'b' })] + modified: [doc('collection/a', 500, { key: 'b' })], + fromCache: true }); } ); diff --git a/packages/firestore/test/unit/specs/spec_test_runner.ts b/packages/firestore/test/unit/specs/spec_test_runner.ts index 85ebf232462..9abd0c82b3f 100644 --- a/packages/firestore/test/unit/specs/spec_test_runner.ts +++ b/packages/firestore/test/unit/specs/spec_test_runner.ts @@ -117,7 +117,7 @@ import { } from './spec_test_components'; import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; import { BundleReader } from '../../../src/util/bundle_reader'; -import { LoadBundleTaskImpl } from '../../../src/core/bundle'; +import { LoadBundleTask } from '../../../src/api/bundle'; import { encodeBase64 } from '../../../src/platform/base64'; import { FakeDocument, @@ -457,12 +457,8 @@ abstract class TestRunner { const reader = new BundleReader( toByteStreamReader(new TextEncoder().encode(bundle)) ); - const task = new LoadBundleTaskImpl(); - this.queue.enqueueAndForget(() => { - return loadBundle(this.syncEngine, reader, task); - }); - - await task; + const task = new LoadBundleTask(); + return this.queue.enqueue(() => loadBundle(this.syncEngine, reader, task)); } private doMutations(mutations: Mutation[]): Promise { From 8fbdd3e3033adc6514f1a6415e4ecc72fb0767c1 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Wed, 1 Jul 2020 18:44:17 -0400 Subject: [PATCH 31/39] Fix bytesPerRead default --- .../firestore/src/platform/browser/byte_stream_reader.ts | 3 +-- packages/firestore/src/platform/byte_stream_reader.ts | 8 +------- packages/firestore/src/util/byte_stream.ts | 9 ++++++++- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/firestore/src/platform/browser/byte_stream_reader.ts b/packages/firestore/src/platform/browser/byte_stream_reader.ts index 0f5a64c6d23..65875d4b225 100644 --- a/packages/firestore/src/platform/browser/byte_stream_reader.ts +++ b/packages/firestore/src/platform/browser/byte_stream_reader.ts @@ -17,14 +17,13 @@ import { BundleSource } from '../../util/bundle_reader'; import { toByteStreamReaderHelper } from '../../util/byte_stream'; -import { DEFAULT_BYTES_PER_READ } from '../byte_stream_reader'; /** * On web, a `ReadableStream` is wrapped around by a `ByteStreamReader`. */ export function toByteStreamReader( source: BundleSource, - bytesPerRead: number = DEFAULT_BYTES_PER_READ + bytesPerRead: number ): ReadableStreamReader { if (source instanceof Uint8Array) { return toByteStreamReaderHelper(source, bytesPerRead); diff --git a/packages/firestore/src/platform/byte_stream_reader.ts b/packages/firestore/src/platform/byte_stream_reader.ts index 2f69db539b8..49aac60e9ff 100644 --- a/packages/firestore/src/platform/byte_stream_reader.ts +++ b/packages/firestore/src/platform/byte_stream_reader.ts @@ -20,13 +20,7 @@ import * as node from './node/byte_stream_reader'; import * as rn from './rn/byte_stream_reader'; import * as browser from './browser/byte_stream_reader'; import { isNode, isReactNative } from '@firebase/util'; - -/** - * For the byte streams where we have control (like backed by a UInt8Array), - * how many bytes to read each time when `ReadableStreamReader.read()` is - * called. - */ -export const DEFAULT_BYTES_PER_READ = 10240; +import { DEFAULT_BYTES_PER_READ } from '../util/byte_stream'; export function toByteStreamReader( source: BundleSource, diff --git a/packages/firestore/src/util/byte_stream.ts b/packages/firestore/src/util/byte_stream.ts index 3434e410270..b24ba040b4d 100644 --- a/packages/firestore/src/util/byte_stream.ts +++ b/packages/firestore/src/util/byte_stream.ts @@ -17,6 +17,13 @@ import { debugAssert } from './assert'; +/** + * For the byte streams where we have control (like backed by a UInt8Array), + * how many bytes to read each time when `ReadableStreamReader.read()` is + * called. + */ +export const DEFAULT_BYTES_PER_READ = 10240; + /** * Builds a `ByteStreamReader` from a UInt8Array. * @param source The data source to use. @@ -25,7 +32,7 @@ import { debugAssert } from './assert'; */ export function toByteStreamReaderHelper( source: Uint8Array, - bytesPerRead: number + bytesPerRead: number = DEFAULT_BYTES_PER_READ ): ReadableStreamReader { debugAssert( bytesPerRead > 0, From 985b205832c3446eb176e567d42a19aba33acfec Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Mon, 6 Jul 2020 15:06:32 -0400 Subject: [PATCH 32/39] Snapshots only once in the end. --- packages/firestore/src/core/bundle.ts | 66 +++++++--------------- packages/firestore/src/core/sync_engine.ts | 30 +++++----- 2 files changed, 36 insertions(+), 60 deletions(-) diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index ee35a7d57df..b43853c8e74 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -166,16 +166,6 @@ export class BundleLoader { private queries: bundleProto.NamedQuery[] = []; /** Batched documents to be saved into storage */ private documents: BundledDocuments = []; - /** - * How many bytes from the bundle are being buffered since last progress - * update. - */ - private bytesBuffered = 0; - /** - * How many documents from the bundle are being buffered since last progress - * update. - */ - private documentsBuffered = 0; /** * A BundleDocumentMetadata is added to the loader, it is saved here while * we wait for the actual document. @@ -196,10 +186,12 @@ export class BundleLoader { * storage, the returned promise will resolve to a `LoadResult`, otherwise * it will resolve to null. */ - addSizedElement(element: SizedBundleElement): Promise { + addSizedElement( + element: SizedBundleElement + ): firestore.LoadBundleTaskProgress | null { debugAssert(!element.isBundleMetadata(), 'Unexpected bundle metadata.'); - this.bytesBuffered += element.byteLength; + this.progress.bytesLoaded += element.byteLength; if (element.payload.namedQuery) { this.queries.push(element.payload.namedQuery); } @@ -212,7 +204,7 @@ export class BundleLoader { metadata: element.payload.documentMetadata, document: undefined }); - this.documentsBuffered += 1; + this.progress.documentsLoaded += 1; } } @@ -225,24 +217,27 @@ export class BundleLoader { metadata: this.unpairedDocumentMetadata!, document: element.payload.document }); - this.documentsBuffered += 1; + this.progress.documentsLoaded += 1; this.unpairedDocumentMetadata = null; } - return this.saveAndReportProgress(); - } - - private async saveAndReportProgress(): Promise { - if ( - this.unpairedDocumentMetadata || - (this.documentsBuffered < - this.progress.totalDocuments * this.thresholdMultiplier && - this.bytesBuffered < - this.progress.totalBytes * this.thresholdMultiplier) - ) { + // Loading a document metadata will not update progress. + if (this.unpairedDocumentMetadata) { return null; } + return { ...this.progress }; + } + + /** + * Update the progress to 'Success' and return the updated progress. + */ + async complete(): Promise { + debugAssert( + !this.unpairedDocumentMetadata, + 'Unexpected document when no pairing metadata is found' + ); + for (const q of this.queries) { await saveNamedQuery(this.localStore, q); } @@ -252,26 +247,7 @@ export class BundleLoader { changedDocs = await applyBundleDocuments(this.localStore, this.documents); } - this.progress.bytesLoaded += this.bytesBuffered; - this.progress.documentsLoaded += this.documentsBuffered; - this.bytesBuffered = 0; - this.documentsBuffered = 0; - this.queries = []; - this.documents = []; - - return new LoadResult({ ...this.progress }, changedDocs); - } - - /** - * Update the progress to 'Success' and return the updated progress. - */ - complete(): firestore.LoadBundleTaskProgress { - debugAssert( - this.queries.length === 0 && this.documents.length === 0, - 'There are more items needs to be saved but complete() was called.' - ); this.progress.taskState = 'Success'; - - return this.progress; + return new LoadResult({ ...this.progress }, changedDocs); } } diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 7f4b1836ccb..71edb14f7ce 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -1423,25 +1423,25 @@ async function loadBundleImpl( !element.payload.metadata, 'Unexpected BundleMetadata element.' ); - const result = await loader.addSizedElement(element); - if (result) { - task._updateProgress(result.progress); - - if (result.changedDocs) { - // eslint-disable-next-line @typescript-eslint/no-floating-promises - syncEngine.emitNewSnapsAndNotifyLocalStore( - result.changedDocs, - /* remoteEvent */ undefined, - /* fromBundle */ true - ); - } + const progress = await loader.addSizedElement(element); + if (progress) { + task._updateProgress(progress); } element = await reader.nextElement(); } - await saveBundle(syncEngine.localStore, metadata); + const result = await loader.complete(); + if (result.changedDocs) { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + syncEngine.emitNewSnapsAndNotifyLocalStore( + result.changedDocs, + /* remoteEvent */ undefined, + /* fromBundle */ true + ); + } - const completeProgress = loader.complete(); - task._completeWith(completeProgress); + // Save metadata, so loading the same bundle will skip. + await saveBundle(syncEngine.localStore, metadata); + task._completeWith(result.progress); } From 685624aa9d78f8dbb679cda98d13b0047abae53c Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Wed, 8 Jul 2020 12:05:42 -0400 Subject: [PATCH 33/39] Temp addressing. --- packages/firestore-types/index.d.ts | 14 +++-- packages/firestore/src/api/bundle.ts | 71 ++++++++++++---------- packages/firestore/src/core/bundle.ts | 23 +++---- packages/firestore/src/core/sync_engine.ts | 12 ++-- 4 files changed, 61 insertions(+), 59 deletions(-) diff --git a/packages/firestore-types/index.d.ts b/packages/firestore-types/index.d.ts index 404ce935912..4f4da1eabfa 100644 --- a/packages/firestore-types/index.d.ts +++ b/packages/firestore-types/index.d.ts @@ -107,12 +107,14 @@ export interface LoadBundleTask { complete?: () => void ): Promise; - then( - onFulfilled?: (a: LoadBundleTaskProgress) => any, - onRejected?: (a: Error) => any - ): Promise; - - catch(onRejected: (a: Error) => any): Promise; + then( + onFulfilled?: (a: LoadBundleTaskProgress) => T | PromiseLike, + onRejected?: (a: Error) => R | PromiseLike + ): Promise; + + catch( + onRejected: (a: Error) => R | PromiseLike + ): Promise; } export interface LoadBundleTaskProgress { diff --git a/packages/firestore/src/api/bundle.ts b/packages/firestore/src/api/bundle.ts index c02f267d4ca..02c8bd75daf 100644 --- a/packages/firestore/src/api/bundle.ts +++ b/packages/firestore/src/api/bundle.ts @@ -17,16 +17,16 @@ import * as firestore from '@firebase/firestore-types'; import { Deferred } from '../util/promise'; +import { PartialObserver } from './observer'; +import { debugAssert } from '../util/assert'; export class LoadBundleTask implements firestore.LoadBundleTask { private _progressResolver = new Deferred(); - private _userProgressHandler?: ( - progress: firestore.LoadBundleTaskProgress - ) => unknown; - private _userProgressErrorHandler?: (err: Error) => unknown; - private _userProgressCompleteHandler?: () => void; + private _progressObserver?: PartialObserver; - private _promiseResolver = new Deferred(); + private _taskCompletionResolver = new Deferred< + firestore.LoadBundleTaskProgress + >(); private _lastProgress: firestore.LoadBundleTaskProgress = { taskState: 'Running', @@ -41,54 +41,58 @@ export class LoadBundleTask implements firestore.LoadBundleTask { error?: (err: Error) => unknown, complete?: () => void ): Promise { - this._userProgressHandler = next; - this._userProgressErrorHandler = error; - this._userProgressCompleteHandler = complete; + this._progressObserver = { + next, + error, + complete + }; return this._progressResolver.promise; } - catch(onRejected: (a: Error) => unknown): Promise { - return this._promiseResolver.promise.catch(onRejected); + catch( + onRejected: (a: Error) => R | PromiseLike + ): Promise { + return this._taskCompletionResolver.promise.catch(onRejected); } - then( - onFulfilled?: (a: firestore.LoadBundleTaskProgress) => unknown, - onRejected?: (a: Error) => unknown - ): Promise { - return this._promiseResolver.promise.then(onFulfilled, onRejected); + then( + onFulfilled?: (a: firestore.LoadBundleTaskProgress) => T | PromiseLike, + onRejected?: (a: Error) => R | PromiseLike + ): Promise { + return this._taskCompletionResolver.promise.then(onFulfilled, onRejected); } /** - * Notifies the completion of loading a bundle, with a provided + * Notifies all observers that bundle loading has completed, with a provided * `LoadBundleTaskProgress` object. */ _completeWith(progress: firestore.LoadBundleTaskProgress): void { this._updateProgress(progress); - if (this._userProgressCompleteHandler) { - this._userProgressCompleteHandler(); + if (this._progressObserver && this._progressObserver.complete) { + this._progressObserver.complete(); } this._progressResolver.resolve(); - this._promiseResolver.resolve(progress); + this._taskCompletionResolver.resolve(progress); } /** - * Notifies a failure of loading a bundle, with a provided `Error` - * as the reason. + * Notifies all observers that bundle loading has failed, with a provided + * `Error` as the reason. */ - _failedWith(error: Error): void { + _failWith(error: Error): void { this._lastProgress.taskState = 'Error'; - if (this._userProgressHandler) { - this._userProgressHandler(this._lastProgress); + if (this._progressObserver && this._progressObserver.next) { + this._progressObserver.next(this._lastProgress); } - if (this._userProgressErrorHandler) { - this._userProgressErrorHandler(error); + if (this._progressObserver && this._progressObserver.error) { + this._progressObserver.error(error); } this._progressResolver.reject(error); - this._promiseResolver.reject(error); + this._taskCompletionResolver.reject(error); } /** @@ -96,13 +100,14 @@ export class LoadBundleTask implements firestore.LoadBundleTask { * @param progress The new progress. */ _updateProgress(progress: firestore.LoadBundleTaskProgress): void { - if (this._lastProgress.taskState === 'Error') { - return; - } + debugAssert( + this._lastProgress.taskState !== 'Error', + 'Cannot update progress on a failed task' + ); this._lastProgress = progress; - if (this._userProgressHandler) { - this._userProgressHandler(progress); + if (this._progressObserver && this._progressObserver.next) { + this._progressObserver.next(progress); } } } diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index b43853c8e74..f84930cf02c 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -112,7 +112,7 @@ export class BundleConverter { * Returns a `LoadBundleTaskProgress` representing the initial progress of * loading a bundle. */ -export function initialProgress( +export function bundleInitialProgress( metadata: BundleMetadata ): firestore.LoadBundleTaskProgress { return { @@ -125,10 +125,10 @@ export function initialProgress( } /** - * Returns a `LoadBundleTaskProgress` representing the progress if the bundle - * is already loaded, and we are skipping current loading. + * Returns a `LoadBundleTaskProgress` representing the progress that the loading + * has succeeded. */ -export function skipLoadingProgress( +export function bundleSuccessProgress( metadata: BundleMetadata ): firestore.LoadBundleTaskProgress { return { @@ -154,14 +154,6 @@ export class LoadResult { export class BundleLoader { /** The current progress of loading */ private progress: firestore.LoadBundleTaskProgress; - /** - * The threshold multiplier used to determine whether enough elements are - * batched to be loaded, and a progress update is needed. - * - * Applies to both `documentsBuffered` and `bytesBuffered`, triggers storage - * update and reports progress when either of them cross the threshold. - */ - private thresholdMultiplier = 0.01; /** Batched queries to be saved into storage */ private queries: bundleProto.NamedQuery[] = []; /** Batched documents to be saved into storage */ @@ -176,15 +168,14 @@ export class BundleLoader { private metadata: bundleProto.BundleMetadata, private localStore: LocalStore ) { - this.progress = initialProgress(metadata); + this.progress = bundleInitialProgress(metadata); } /** * Adds an element from the bundle to the loader. * - * If adding this element leads to actually saving the batched elements into - * storage, the returned promise will resolve to a `LoadResult`, otherwise - * it will resolve to null. + * Returns a new progress if adding the element leads to a new progress, + * otherwise returns null. */ addSizedElement( element: SizedBundleElement diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 71edb14f7ce..c8e8d4032a2 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -77,7 +77,11 @@ import { ViewSnapshot } from './view_snapshot'; import { AsyncQueue, wrapInUserErrorIfRecoverable } from '../util/async_queue'; import { TransactionRunner } from './transaction_runner'; import { BundleReader } from '../util/bundle_reader'; -import { BundleLoader, initialProgress, skipLoadingProgress } from './bundle'; +import { + BundleLoader, + bundleInitialProgress, + bundleSuccessProgress +} from './bundle'; import { Datastore } from '../remote/datastore'; import { LoadBundleTask } from '../api/bundle'; @@ -1397,7 +1401,7 @@ export async function loadBundle( try { await loadBundleImpl(syncEngineImpl, bundleReader, task); } catch (e) { - task._failedWith(e); + task._failWith(e); } } @@ -1410,11 +1414,11 @@ async function loadBundleImpl( const skip = await hasNewerBundle(syncEngine.localStore, metadata); if (skip) { await reader.close(); - task._completeWith(skipLoadingProgress(metadata)); + task._completeWith(bundleSuccessProgress(metadata)); return; } - task._updateProgress(initialProgress(metadata)); + task._updateProgress(bundleInitialProgress(metadata)); const loader = new BundleLoader(metadata, syncEngine.localStore); let element = await reader.nextElement(); From 55769636263f00bdeca5fbe983cff3ec85911df4 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Wed, 8 Jul 2020 12:29:33 -0400 Subject: [PATCH 34/39] Apply bundle.ts patch --- packages/firestore/src/core/bundle.ts | 62 ++++++++++------------ packages/firestore/src/core/sync_engine.ts | 2 +- 2 files changed, 29 insertions(+), 35 deletions(-) diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index f84930cf02c..08a28c129e3 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -67,7 +67,7 @@ export interface NamedQuery { */ interface BundledDocument { metadata: bundleProto.BundledDocumentMetadata; - document: api.Document | undefined; + document?: api.Document; } /** @@ -158,11 +158,6 @@ export class BundleLoader { private queries: bundleProto.NamedQuery[] = []; /** Batched documents to be saved into storage */ private documents: BundledDocuments = []; - /** - * A BundleDocumentMetadata is added to the loader, it is saved here while - * we wait for the actual document. - */ - private unpairedDocumentMetadata: bundleProto.BundledDocumentMetadata | null = null; constructor( private metadata: bundleProto.BundleMetadata, @@ -183,50 +178,49 @@ export class BundleLoader { debugAssert(!element.isBundleMetadata(), 'Unexpected bundle metadata.'); this.progress.bytesLoaded += element.byteLength; + + let documentsLoaded = this.progress.documentsLoaded; + if (element.payload.namedQuery) { this.queries.push(element.payload.namedQuery); - } - - if (element.payload.documentMetadata) { - if (element.payload.documentMetadata.exists) { - this.unpairedDocumentMetadata = element.payload.documentMetadata; - } else { - this.documents.push({ - metadata: element.payload.documentMetadata, - document: undefined - }); - this.progress.documentsLoaded += 1; + } else if (element.payload.documentMetadata) { + this.documents.push({ metadata: element.payload.documentMetadata }); + if (!element.payload.documentMetadata.exists) { + ++documentsLoaded; } - } - - if (element.payload.document) { + } else if (element.payload.document) { debugAssert( - !!this.unpairedDocumentMetadata, - 'Unexpected document when no pairing metadata is found' + this.documents.length > 0 && + this.documents[this.documents.length - 1].metadata.name == + element.payload.document.name, + 'The document being added does not match the stored metadata.' ); - this.documents.push({ - metadata: this.unpairedDocumentMetadata!, - document: element.payload.document - }); - this.progress.documentsLoaded += 1; - this.unpairedDocumentMetadata = null; + this.documents[this.documents.length - 1].document = + element.payload.document; + ++documentsLoaded; } - // Loading a document metadata will not update progress. - if (this.unpairedDocumentMetadata) { - return null; + if (documentsLoaded !== this.progress.documentsLoaded) { + this.progress.documentsLoaded = documentsLoaded; + return { ...this.progress }; } - return { ...this.progress }; + return null; } /** * Update the progress to 'Success' and return the updated progress. */ async complete(): Promise { + const lastDocument = + this.documents.length === 0 + ? null + : this.documents[this.documents.length - 1]; debugAssert( - !this.unpairedDocumentMetadata, - 'Unexpected document when no pairing metadata is found' + !!lastDocument || + !lastDocument!.metadata.exists || + (!!lastDocument!.metadata.exists && !!lastDocument!.document), + 'Bundled documents ends with a document metadata.' ); for (const q of this.queries) { diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index c8e8d4032a2..50a89c3dc51 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -1384,7 +1384,7 @@ export function newMultiTabSyncEngine( } /** - * Loads a Firestore bundle into the SDK, the returned promise resolves when + * Loads a Firestore bundle into the SDK. The returned promise resolves when * the bundle finished loading. * * @param bundleReader Bundle to load into the SDK. From 85e3ac643e5c64633a4a17013a9079cd03a62d91 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Wed, 8 Jul 2020 14:14:40 -0400 Subject: [PATCH 35/39] Change how task is passed. --- packages/firestore/src/api/bundle.ts | 5 +- .../firestore/src/core/firestore_client.ts | 9 ++- packages/firestore/src/core/sync_engine.ts | 79 ++++++++++--------- .../test/unit/specs/spec_test_runner.ts | 8 +- 4 files changed, 57 insertions(+), 44 deletions(-) diff --git a/packages/firestore/src/api/bundle.ts b/packages/firestore/src/api/bundle.ts index 02c8bd75daf..d6bb716d903 100644 --- a/packages/firestore/src/api/bundle.ts +++ b/packages/firestore/src/api/bundle.ts @@ -20,7 +20,10 @@ import { Deferred } from '../util/promise'; import { PartialObserver } from './observer'; import { debugAssert } from '../util/assert'; -export class LoadBundleTask implements firestore.LoadBundleTask { +export class LoadBundleTask + implements + firestore.LoadBundleTask, + PromiseLike { private _progressResolver = new Deferred(); private _progressObserver?: PartialObserver; diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index 0c42d132d1b..80e60696ebb 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -27,7 +27,7 @@ import { newDatastore } from '../remote/datastore'; import { RemoteStore } from '../remote/remote_store'; import { AsyncQueue, wrapInUserErrorIfRecoverable } from '../util/async_queue'; import { Code, FirestoreError } from '../util/error'; -import { logDebug } from '../util/log'; +import { logDebug, logWarn } from '../util/log'; import { Deferred } from '../util/promise'; import { EventManager, @@ -530,8 +530,11 @@ export class FirestoreClient { } const reader = new BundleReader(toByteStreamReader(content)); const task = new LoadBundleTask(); - this.asyncQueue.enqueueAndForget(() => { - return loadBundle(this.syncEngine, reader, task); + this.asyncQueue.enqueueAndForget(async () => { + loadBundle(this.syncEngine, reader, task); + return task.catch(e => { + logWarn(`Loading bundle failed with ${e}`); + }); }); return task; diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 50a89c3dc51..5c3eefb960c 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -1390,19 +1390,16 @@ export function newMultiTabSyncEngine( * @param bundleReader Bundle to load into the SDK. * @param task LoadBundleTask used to update the loading progress to public API. */ -export async function loadBundle( +export function loadBundle( syncEngine: SyncEngine, bundleReader: BundleReader, task: LoadBundleTask -): Promise { +): void { const syncEngineImpl = debugCast(syncEngine, SyncEngineImpl); syncEngineImpl.assertSubscribed('loadBundle()'); - try { - await loadBundleImpl(syncEngineImpl, bundleReader, task); - } catch (e) { - task._failWith(e); - } + // tslint:disable-next-line:no-floating-promises + loadBundleImpl(syncEngineImpl, bundleReader, task); } async function loadBundleImpl( @@ -1410,42 +1407,46 @@ async function loadBundleImpl( reader: BundleReader, task: LoadBundleTask ): Promise { - const metadata = await reader.getMetadata(); - const skip = await hasNewerBundle(syncEngine.localStore, metadata); - if (skip) { - await reader.close(); - task._completeWith(bundleSuccessProgress(metadata)); - return; - } + try { + const metadata = await reader.getMetadata(); + const skip = await hasNewerBundle(syncEngine.localStore, metadata); + if (skip) { + await reader.close(); + task._completeWith(bundleSuccessProgress(metadata)); + return; + } - task._updateProgress(bundleInitialProgress(metadata)); + task._updateProgress(bundleInitialProgress(metadata)); - const loader = new BundleLoader(metadata, syncEngine.localStore); - let element = await reader.nextElement(); - while (element) { - debugAssert( - !element.payload.metadata, - 'Unexpected BundleMetadata element.' - ); - const progress = await loader.addSizedElement(element); - if (progress) { - task._updateProgress(progress); + const loader = new BundleLoader(metadata, syncEngine.localStore); + let element = await reader.nextElement(); + while (element) { + debugAssert( + !element.payload.metadata, + 'Unexpected BundleMetadata element.' + ); + const progress = await loader.addSizedElement(element); + if (progress) { + task._updateProgress(progress); + } + + element = await reader.nextElement(); } - element = await reader.nextElement(); - } + const result = await loader.complete(); + if (result.changedDocs) { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + syncEngine.emitNewSnapsAndNotifyLocalStore( + result.changedDocs, + /* remoteEvent */ undefined, + /* fromBundle */ true + ); + } - const result = await loader.complete(); - if (result.changedDocs) { - // eslint-disable-next-line @typescript-eslint/no-floating-promises - syncEngine.emitNewSnapsAndNotifyLocalStore( - result.changedDocs, - /* remoteEvent */ undefined, - /* fromBundle */ true - ); + // Save metadata, so loading the same bundle will skip. + await saveBundle(syncEngine.localStore, metadata); + task._completeWith(result.progress); + } catch (e) { + task._failWith(e); } - - // Save metadata, so loading the same bundle will skip. - await saveBundle(syncEngine.localStore, metadata); - task._completeWith(result.progress); } diff --git a/packages/firestore/test/unit/specs/spec_test_runner.ts b/packages/firestore/test/unit/specs/spec_test_runner.ts index 9abd0c82b3f..af3780bf265 100644 --- a/packages/firestore/test/unit/specs/spec_test_runner.ts +++ b/packages/firestore/test/unit/specs/spec_test_runner.ts @@ -125,6 +125,7 @@ import { testWindow } from '../../util/test_platform'; import { toByteStreamReader } from '../../../src/platform/byte_stream_reader'; +import { logWarn } from '../../../src/util/log'; const ARBITRARY_SEQUENCE_NUMBER = 2; @@ -458,7 +459,12 @@ abstract class TestRunner { toByteStreamReader(new TextEncoder().encode(bundle)) ); const task = new LoadBundleTask(); - return this.queue.enqueue(() => loadBundle(this.syncEngine, reader, task)); + return this.queue.enqueue(async () => { + loadBundle(this.syncEngine, reader, task); + await task.catch(e => { + logWarn(`Loading bundle failed with ${e}`); + }); + }); } private doMutations(mutations: Mutation[]): Promise { From 2f639aeee3d6e0257b1d14103b11f6579e8aaa14 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Wed, 8 Jul 2020 14:51:50 -0400 Subject: [PATCH 36/39] From promise to void --- packages/firestore-types/index.d.ts | 2 +- packages/firestore/src/api/bundle.ts | 7 +---- packages/firestore/src/core/bundle.ts | 2 +- packages/firestore/src/core/sync_engine.ts | 5 ++- .../integration/api_internal/bundle.test.ts | 31 ++++++++++--------- 5 files changed, 21 insertions(+), 26 deletions(-) diff --git a/packages/firestore-types/index.d.ts b/packages/firestore-types/index.d.ts index 4f4da1eabfa..9b2bc043cce 100644 --- a/packages/firestore-types/index.d.ts +++ b/packages/firestore-types/index.d.ts @@ -105,7 +105,7 @@ export interface LoadBundleTask { next?: (progress: LoadBundleTaskProgress) => any, error?: (error: Error) => any, complete?: () => void - ): Promise; + ): void; then( onFulfilled?: (a: LoadBundleTaskProgress) => T | PromiseLike, diff --git a/packages/firestore/src/api/bundle.ts b/packages/firestore/src/api/bundle.ts index d6bb716d903..2aebf730587 100644 --- a/packages/firestore/src/api/bundle.ts +++ b/packages/firestore/src/api/bundle.ts @@ -24,9 +24,7 @@ export class LoadBundleTask implements firestore.LoadBundleTask, PromiseLike { - private _progressResolver = new Deferred(); private _progressObserver?: PartialObserver; - private _taskCompletionResolver = new Deferred< firestore.LoadBundleTaskProgress >(); @@ -43,13 +41,12 @@ export class LoadBundleTask next?: (progress: firestore.LoadBundleTaskProgress) => unknown, error?: (err: Error) => unknown, complete?: () => void - ): Promise { + ): void { this._progressObserver = { next, error, complete }; - return this._progressResolver.promise; } catch( @@ -74,7 +71,6 @@ export class LoadBundleTask if (this._progressObserver && this._progressObserver.complete) { this._progressObserver.complete(); } - this._progressResolver.resolve(); this._taskCompletionResolver.resolve(progress); } @@ -93,7 +89,6 @@ export class LoadBundleTask if (this._progressObserver && this._progressObserver.error) { this._progressObserver.error(error); } - this._progressResolver.reject(error); this._taskCompletionResolver.reject(error); } diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index 08a28c129e3..66b9c88c640 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -191,7 +191,7 @@ export class BundleLoader { } else if (element.payload.document) { debugAssert( this.documents.length > 0 && - this.documents[this.documents.length - 1].metadata.name == + this.documents[this.documents.length - 1].metadata.name === element.payload.document.name, 'The document being added does not match the stored metadata.' ); diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 5c3eefb960c..9a2c55615b7 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -1398,7 +1398,7 @@ export function loadBundle( const syncEngineImpl = debugCast(syncEngine, SyncEngineImpl); syncEngineImpl.assertSubscribed('loadBundle()'); - // tslint:disable-next-line:no-floating-promises + // eslint-disable-next-line @typescript-eslint/no-floating-promises loadBundleImpl(syncEngineImpl, bundleReader, task); } @@ -1435,8 +1435,7 @@ async function loadBundleImpl( const result = await loader.complete(); if (result.changedDocs) { - // eslint-disable-next-line @typescript-eslint/no-floating-promises - syncEngine.emitNewSnapsAndNotifyLocalStore( + await syncEngine.emitNewSnapsAndNotifyLocalStore( result.changedDocs, /* remoteEvent */ undefined, /* fromBundle */ true diff --git a/packages/firestore/test/integration/api_internal/bundle.test.ts b/packages/firestore/test/integration/api_internal/bundle.test.ts index 0cc51c4dad1..9b784c4c741 100644 --- a/packages/firestore/test/integration/api_internal/bundle.test.ts +++ b/packages/firestore/test/integration/api_internal/bundle.test.ts @@ -94,7 +94,7 @@ apiDescribe('Bundles', (persistence: boolean) => { const task = db.loadBundle( builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ); - await task.onProgress( + task.onProgress( progress => { progressEvents.push(progress); }, @@ -103,6 +103,7 @@ apiDescribe('Bundles', (persistence: boolean) => { completeCalled = true; } ); + await task; let fulfillProgress: firestore.LoadBundleTaskProgress; await task.then(progress => { fulfillProgress = progress; @@ -150,21 +151,21 @@ apiDescribe('Bundles', (persistence: boolean) => { let completeCalled = false; const progressEvents: firestore.LoadBundleTaskProgress[] = []; - await db - .loadBundle( - encoder.encode( - builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) - ) + const task = db.loadBundle( + encoder.encode( + builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ) - .onProgress( - progress => { - progressEvents.push(progress); - }, - error => {}, - () => { - completeCalled = true; - } - ); + ); + task.onProgress( + progress => { + progressEvents.push(progress); + }, + error => {}, + () => { + completeCalled = true; + } + ); + await task; expect(completeCalled).to.be.true; // No loading actually happened in the second `loadBundle` call only the From 57a1c6395de300e6e63b653eb80ad7504353b3f2 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Thu, 9 Jul 2020 15:32:17 -0400 Subject: [PATCH 37/39] Spec tests comments --- packages/firestore/src/core/sync_engine.ts | 12 +-- packages/firestore/src/core/view.ts | 6 +- packages/firestore/src/util/byte_stream.ts | 6 +- .../integration/api_internal/bundle.test.ts | 2 +- .../test/unit/specs/bundle_spec.test.ts | 95 +++++++++++-------- 5 files changed, 69 insertions(+), 52 deletions(-) diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 9a2c55615b7..603cf2a514a 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -848,8 +848,7 @@ class SyncEngineImpl implements SyncEngine { async emitNewSnapsAndNotifyLocalStore( changes: MaybeDocumentMap, - remoteEvent?: RemoteEvent, - fromBundle: boolean = false + remoteEvent?: RemoteEvent ): Promise { const newSnaps: ViewSnapshot[] = []; const docChangesInAllViews: LocalViewChanges[] = []; @@ -881,8 +880,7 @@ class SyncEngineImpl implements SyncEngine { const viewChange = queryView.view.applyChanges( viewDocChanges, /* updateLimboDocuments= */ this.isPrimaryClient, - targetChange, - fromBundle + targetChange ); this.updateTrackedLimbos( queryView.targetId, @@ -1435,10 +1433,12 @@ async function loadBundleImpl( const result = await loader.complete(); if (result.changedDocs) { + // TODO(b/160876443): This currently raises snapshots with + // `fromCache=false` if users already listen to some queries and bundles + // has newer version. await syncEngine.emitNewSnapsAndNotifyLocalStore( result.changedDocs, - /* remoteEvent */ undefined, - /* fromBundle */ true + /* remoteEvent */ undefined ); } diff --git a/packages/firestore/src/core/view.ts b/packages/firestore/src/core/view.ts index 1cec1fcb522..c9f34f08103 100644 --- a/packages/firestore/src/core/view.ts +++ b/packages/firestore/src/core/view.ts @@ -280,15 +280,13 @@ export class View { * change. * @param targetChange A target change to apply for computing limbo docs and * sync state. - * @param fromBundle Whether the changes are from applying a bundle file. * @return A new ViewChange with the given docs, changes, and sync state. */ // PORTING NOTE: The iOS/Android clients always compute limbo document changes. applyChanges( docChanges: ViewDocumentChanges, updateLimboDocuments: boolean, - targetChange?: TargetChange, - fromBundle: boolean = false + targetChange?: TargetChange ): ViewChange { debugAssert( !docChanges.needsRefill, @@ -325,7 +323,7 @@ export class View { oldDocs, changes, docChanges.mutatedKeys, - newSyncState === SyncState.Local || fromBundle, + newSyncState === SyncState.Local, syncStateChanged, /* excludesMetadataChanges= */ false ); diff --git a/packages/firestore/src/util/byte_stream.ts b/packages/firestore/src/util/byte_stream.ts index b24ba040b4d..a2723a82528 100644 --- a/packages/firestore/src/util/byte_stream.ts +++ b/packages/firestore/src/util/byte_stream.ts @@ -18,9 +18,9 @@ import { debugAssert } from './assert'; /** - * For the byte streams where we have control (like backed by a UInt8Array), - * how many bytes to read each time when `ReadableStreamReader.read()` is - * called. + * How many bytes to read each time when `ReadableStreamReader.read()` is + * called. Only applicable for byte streams that we control (e.g. those backed + * by an UInt8Array). */ export const DEFAULT_BYTES_PER_READ = 10240; diff --git a/packages/firestore/test/integration/api_internal/bundle.test.ts b/packages/firestore/test/integration/api_internal/bundle.test.ts index 9b784c4c741..b0e18b5b1cf 100644 --- a/packages/firestore/test/integration/api_internal/bundle.test.ts +++ b/packages/firestore/test/integration/api_internal/bundle.test.ts @@ -216,7 +216,7 @@ apiDescribe('Bundles', (persistence: boolean) => { let builder = bundleWithTestDocs(db); return withAlternateTestDb(persistence, async otherDb => { // eslint-disable-next-line @typescript-eslint/no-floating-promises - expect( + await expect( otherDb.loadBundle( builder.build('test-bundle', { seconds: 1001, nanos: 9999 }) ) diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts index f1ad153488d..40faa6fa5ea 100644 --- a/packages/firestore/test/unit/specs/bundle_spec.test.ts +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -16,7 +16,13 @@ */ import { Query } from '../../../src/core/query'; -import { doc, path, TestSnapshotVersion, version } from '../../util/helpers'; +import { + doc, + path, + TestSnapshotVersion, + version, + wrapObject +} from '../../util/helpers'; import { describeSpec, specTest } from './describe_spec'; import { client, spec } from './spec_builder'; @@ -26,17 +32,17 @@ import { TEST_DATABASE_ID } from '../local/persistence_test_helpers'; import { DocumentKey } from '../../../src/model/document_key'; -import * as api from '../../../src/protos/firestore_proto_api'; -import { Value } from '../../../src/protos/firestore_proto_api'; import { toVersion } from '../../../src/remote/serializer'; +import { JsonObject } from '../../../src/model/object_value'; interface TestBundleDocument { key: DocumentKey; readTime: TestSnapshotVersion; createTime?: TestSnapshotVersion; updateTime?: TestSnapshotVersion; - content?: api.ApiClientObjectMap; + content?: JsonObject; } + function bundleWithDocument(testDoc: TestBundleDocument): string { const builder = new TestBundleBuilder(TEST_DATABASE_ID); builder.addDocumentMetadata( @@ -49,7 +55,7 @@ function bundleWithDocument(testDoc: TestBundleDocument): string { testDoc.key, toVersion(JSON_SERIALIZER, version(testDoc.createTime)), toVersion(JSON_SERIALIZER, version(testDoc.updateTime!)), - testDoc.content! + wrapObject(testDoc.content!).proto.mapValue.fields! ); } return builder.build( @@ -69,7 +75,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { readTime: 3000, createTime: 1999, updateTime: 2999, - content: { key: { stringValue: 'b' } } + content: { key: 'b' } }); return spec() @@ -77,25 +83,29 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { .watchAcksFull(query1, 1000, docA) .expectEvents(query1, { added: [docA] }) .loadBundle(bundleString) - .expectEvents(query1, { modified: [docAChanged], fromCache: true }); + .expectEvents(query1, { modified: [docAChanged] }); }); - specTest('Newer deleted docs from bundles should delete cache', [], () => { - const query1 = Query.atPath(path('collection')); - const docA = doc('collection/a', 1000, { key: 'a' }); + specTest( + 'Newer deleted docs from bundles should delete cache docs', + [], + () => { + const query1 = Query.atPath(path('collection')); + const docA = doc('collection/a', 1000, { key: 'a' }); - const bundleString = bundleWithDocument({ - key: docA.key, - readTime: 3000 - }); + const bundleString = bundleWithDocument({ + key: docA.key, + readTime: 3000 + }); - return spec() - .userListens(query1) - .watchAcksFull(query1, 1000, docA) - .expectEvents(query1, { added: [docA] }) - .loadBundle(bundleString) - .expectEvents(query1, { removed: [docA], fromCache: true }); - }); + return spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA) + .expectEvents(query1, { added: [docA] }) + .loadBundle(bundleString) + .expectEvents(query1, { removed: [docA] }); + } + ); specTest('Older deleted docs from bundles should do nothing', [], () => { const query1 = Query.atPath(path('collection')); @@ -117,7 +127,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { }); specTest( - 'Newer docs from bundles should raise snapshot only when watch catches up with acknowledged writes', + 'Newer docs from bundles should raise snapshot only when Watch catches up with acknowledged writes', [], () => { const query = Query.atPath(path('collection')); @@ -128,7 +138,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { readTime: 500, createTime: 250, updateTime: 500, - content: { key: { stringValue: 'b' } } + content: { key: 'b' } }); const bundleAfterMutationAck = bundleWithDocument({ @@ -136,10 +146,12 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { readTime: 1001, createTime: 250, updateTime: 1001, - content: { key: { stringValue: 'fromBundle' } } + content: { key: 'fromBundle' } }); return ( spec() + // TODO(b/160878667): Figure out what happens when memory eager GC is on + // a bundle is loaded. .withGCEnabled(false) .userListens(query) .watchAcksFull(query, 250, docA) @@ -166,8 +178,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { // the acknowledged mutation. .loadBundle(bundleAfterMutationAck) .expectEvents(query, { - modified: [doc('collection/a', 1001, { key: 'fromBundle' })], - fromCache: true + modified: [doc('collection/a', 1001, { key: 'fromBundle' })] }) ); } @@ -185,7 +196,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { readTime: 1001, createTime: 250, updateTime: 1001, - content: { key: { stringValue: 'fromBundle' } } + content: { key: 'fromBundle' } }); return ( @@ -223,8 +234,9 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { readTime: 500, createTime: 250, updateTime: 500, - content: { key: { stringValue: 'b' } } + content: { key: 'b' } }); + const limboQuery = Query.atPath(docA.key.path); return ( spec() @@ -235,11 +247,21 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { .expectEvents(query, {}) // Bundle tells otherwise, leads to limbo. .loadBundle(bundleString1) + .expectLimboDocs(docA.key) .expectEvents(query, { added: [doc('collection/a', 500, { key: 'b' })], fromCache: true }) - .expectLimboDocs(docA.key) + // .watchAcksFull(limboQuery, 1002, docA1) + .watchAcks(limboQuery) + .watchSends({ affects: [limboQuery] }) + .watchCurrents(limboQuery, 'resume-token-1002') + .watchSnapshots(1002) + .expectLimboDocs() + .expectEvents(query, { + removed: [doc('collection/a', 500, { key: 'b' })], + fromCache: false + }) ); }); @@ -254,7 +276,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { readTime: 500, createTime: 250, updateTime: 500, - content: { key: { stringValue: 'b' } } + content: { key: 'b' } }); return client(0) @@ -286,7 +308,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { readTime: 500, createTime: 250, updateTime: 500, - content: { key: { stringValue: 'b' } } + content: { key: 'b' } }); return client(0) @@ -302,8 +324,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { }) .loadBundle(bundleString1) .expectEvents(query, { - modified: [doc('collection/a', 500, { key: 'b' })], - fromCache: true + modified: [doc('collection/a', 500, { key: 'b' })] }); } ); @@ -319,7 +340,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { readTime: 500, createTime: 250, updateTime: 500, - content: { key: { stringValue: 'b' } } + content: { key: 'b' } }); return client(0) @@ -336,13 +357,11 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { .client(0) .loadBundle(bundleString1) .expectEvents(query, { - modified: [doc('collection/a', 500, { key: 'b' })], - fromCache: true + modified: [doc('collection/a', 500, { key: 'b' })] }) .client(1) .expectEvents(query, { - modified: [doc('collection/a', 500, { key: 'b' })], - fromCache: true + modified: [doc('collection/a', 500, { key: 'b' })] }); } ); From b6b261bf495e3cacec212d5a6ea28714a9d818d9 Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 10 Jul 2020 18:48:15 -0400 Subject: [PATCH 38/39] More feedbacks. --- packages/firestore/src/api/bundle.ts | 20 +++-- packages/firestore/src/core/bundle.ts | 25 +++--- .../firestore/src/core/firestore_client.ts | 2 +- .../test/unit/specs/bundle_spec.test.ts | 89 ++++++++++--------- 4 files changed, 71 insertions(+), 65 deletions(-) diff --git a/packages/firestore/src/api/bundle.ts b/packages/firestore/src/api/bundle.ts index 2aebf730587..703a5d06482 100644 --- a/packages/firestore/src/api/bundle.ts +++ b/packages/firestore/src/api/bundle.ts @@ -24,7 +24,9 @@ export class LoadBundleTask implements firestore.LoadBundleTask, PromiseLike { - private _progressObserver?: PartialObserver; + private _progressObserver: PartialObserver< + firestore.LoadBundleTaskProgress + > = {}; private _taskCompletionResolver = new Deferred< firestore.LoadBundleTaskProgress >(); @@ -67,8 +69,12 @@ export class LoadBundleTask * `LoadBundleTaskProgress` object. */ _completeWith(progress: firestore.LoadBundleTaskProgress): void { + debugAssert( + progress.taskState === 'Success', + 'Task is not completed with Success.' + ); this._updateProgress(progress); - if (this._progressObserver && this._progressObserver.complete) { + if (this._progressObserver.complete) { this._progressObserver.complete(); } @@ -82,11 +88,11 @@ export class LoadBundleTask _failWith(error: Error): void { this._lastProgress.taskState = 'Error'; - if (this._progressObserver && this._progressObserver.next) { + if (this._progressObserver.next) { this._progressObserver.next(this._lastProgress); } - if (this._progressObserver && this._progressObserver.error) { + if (this._progressObserver.error) { this._progressObserver.error(error); } @@ -99,12 +105,12 @@ export class LoadBundleTask */ _updateProgress(progress: firestore.LoadBundleTaskProgress): void { debugAssert( - this._lastProgress.taskState !== 'Error', - 'Cannot update progress on a failed task' + this._lastProgress.taskState === 'Running', + 'Cannot update progress on a completed or failed task' ); this._lastProgress = progress; - if (this._progressObserver && this._progressObserver.next) { + if (this._progressObserver.next) { this._progressObserver.next(progress); } } diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index 66b9c88c640..6e4bbfb0efe 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -140,7 +140,7 @@ export function bundleSuccessProgress( }; } -export class LoadResult { +export class BundleLoadResult { constructor( readonly progress: firestore.LoadBundleTaskProgress, readonly changedDocs?: MaybeDocumentMap @@ -211,28 +211,23 @@ export class BundleLoader { /** * Update the progress to 'Success' and return the updated progress. */ - async complete(): Promise { - const lastDocument = - this.documents.length === 0 - ? null - : this.documents[this.documents.length - 1]; + async complete(): Promise { debugAssert( - !!lastDocument || - !lastDocument!.metadata.exists || - (!!lastDocument!.metadata.exists && !!lastDocument!.document), - 'Bundled documents ends with a document metadata.' + this.documents[this.documents.length - 1]?.metadata.exists !== true || + !!this.documents[this.documents.length - 1].document, + 'Bundled documents ends with a document metadata and missing document.' ); for (const q of this.queries) { await saveNamedQuery(this.localStore, q); } - let changedDocs; - if (this.documents.length > 0) { - changedDocs = await applyBundleDocuments(this.localStore, this.documents); - } + const changedDocs = await applyBundleDocuments( + this.localStore, + this.documents + ); this.progress.taskState = 'Success'; - return new LoadResult({ ...this.progress }, changedDocs); + return new BundleLoadResult({ ...this.progress }, changedDocs); } } diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index 80e60696ebb..533e8bd4a75 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -533,7 +533,7 @@ export class FirestoreClient { this.asyncQueue.enqueueAndForget(async () => { loadBundle(this.syncEngine, reader, task); return task.catch(e => { - logWarn(`Loading bundle failed with ${e}`); + logWarn(LOG_TAG, `Loading bundle failed with ${e}`); }); }); diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts index 40faa6fa5ea..fa7b08d8c45 100644 --- a/packages/firestore/test/unit/specs/bundle_spec.test.ts +++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts @@ -67,31 +67,36 @@ function bundleWithDocument(testDoc: TestBundleDocument): string { describeSpec('Bundles:', ['no-ios', 'no-android'], () => { specTest('Newer docs from bundles should overwrite cache', [], () => { const query1 = Query.atPath(path('collection')); - const docA = doc('collection/a', 1000, { key: 'a' }); - const docAChanged = doc('collection/a', 2999, { key: 'b' }); + const docA = doc('collection/a', 1000, { value: 'a' }); + const docAChanged = doc('collection/a', 2999, { value: 'b' }); const bundleString = bundleWithDocument({ key: docA.key, readTime: 3000, createTime: 1999, updateTime: 2999, - content: { key: 'b' } + content: { value: 'b' } }); - return spec() - .userListens(query1) - .watchAcksFull(query1, 1000, docA) - .expectEvents(query1, { added: [docA] }) - .loadBundle(bundleString) - .expectEvents(query1, { modified: [docAChanged] }); + return ( + spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA) + .expectEvents(query1, { added: [docA] }) + // TODO(b/160876443): This currently raises snapshots with + // `fromCache=false` if users already listen to some queries and bundles + // has newer version. + .loadBundle(bundleString) + .expectEvents(query1, { modified: [docAChanged] }) + ); }); specTest( - 'Newer deleted docs from bundles should delete cache docs', + 'Newer deleted docs from bundles should delete cached docs', [], () => { const query1 = Query.atPath(path('collection')); - const docA = doc('collection/a', 1000, { key: 'a' }); + const docA = doc('collection/a', 1000, { value: 'a' }); const bundleString = bundleWithDocument({ key: docA.key, @@ -109,7 +114,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { specTest('Older deleted docs from bundles should do nothing', [], () => { const query1 = Query.atPath(path('collection')); - const docA = doc('collection/a', 1000, { key: 'a' }); + const docA = doc('collection/a', 1000, { value: 'a' }); const bundleString = bundleWithDocument({ key: docA.key, @@ -131,14 +136,14 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { [], () => { const query = Query.atPath(path('collection')); - const docA = doc('collection/a', 250, { key: 'a' }); + const docA = doc('collection/a', 250, { value: 'a' }); const bundleBeforeMutationAck = bundleWithDocument({ key: docA.key, readTime: 500, createTime: 250, updateTime: 500, - content: { key: 'b' } + content: { value: 'b' } }); const bundleAfterMutationAck = bundleWithDocument({ @@ -146,7 +151,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { readTime: 1001, createTime: 250, updateTime: 1001, - content: { key: 'fromBundle' } + content: { value: 'fromBundle' } }); return ( spec() @@ -156,29 +161,29 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { .userListens(query) .watchAcksFull(query, 250, docA) .expectEvents(query, { - added: [doc('collection/a', 250, { key: 'a' })] + added: [doc('collection/a', 250, { value: 'a' })] }) - .userPatches('collection/a', { key: 'patched' }) + .userPatches('collection/a', { value: 'patched' }) .expectEvents(query, { modified: [ doc( 'collection/a', 250, - { key: 'patched' }, + { value: 'patched' }, { hasLocalMutations: true } ) ], hasPendingWrites: true }) .writeAcks('collection/a', 1000) - // loading bundleBeforeMutationAck will not raise snapshots, because it is before - // the acknowledged mutation. + // loading bundleBeforeMutationAck will not raise snapshots, because its + // snapshot version is older than the acknowledged mutation. .loadBundle(bundleBeforeMutationAck) // loading bundleAfterMutationAck will raise a snapshot, because it is after // the acknowledged mutation. .loadBundle(bundleAfterMutationAck) .expectEvents(query, { - modified: [doc('collection/a', 1001, { key: 'fromBundle' })] + modified: [doc('collection/a', 1001, { value: 'fromBundle' })] }) ); } @@ -189,14 +194,14 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { [], () => { const query = Query.atPath(path('collection')); - const docA = doc('collection/a', 250, { key: 'a' }); + const docA = doc('collection/a', 250, { value: 'a' }); const bundleString = bundleWithDocument({ key: docA.key, readTime: 1001, createTime: 250, updateTime: 1001, - content: { key: 'fromBundle' } + content: { value: 'fromBundle' } }); return ( @@ -205,22 +210,22 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { .userListens(query) .watchAcksFull(query, 250, docA) .expectEvents(query, { - added: [doc('collection/a', 250, { key: 'a' })] + added: [doc('collection/a', 250, { value: 'a' })] }) - .userPatches('collection/a', { key: 'patched' }) + .userPatches('collection/a', { value: 'patched' }) .expectEvents(query, { modified: [ doc( 'collection/a', 250, - { key: 'patched' }, + { value: 'patched' }, { hasLocalMutations: true } ) ], hasPendingWrites: true }) // Loading the bundle will not raise snapshots, because the - // mutation is not acknowledged. + // mutation has not been acknowledged. .loadBundle(bundleString) ); } @@ -228,13 +233,13 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { specTest('Newer docs from bundles might lead to limbo doc', [], () => { const query = Query.atPath(path('collection')); - const docA = doc('collection/a', 1000, { key: 'a' }); + const docA = doc('collection/a', 1000, { value: 'a' }); const bundleString1 = bundleWithDocument({ key: docA.key, readTime: 500, createTime: 250, updateTime: 500, - content: { key: 'b' } + content: { value: 'b' } }); const limboQuery = Query.atPath(docA.key.path); @@ -243,13 +248,13 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { .withGCEnabled(false) .userListens(query) .watchAcksFull(query, 250) - // Backend tells there is no such doc. + // Backend tells is there is no such doc. .expectEvents(query, {}) // Bundle tells otherwise, leads to limbo. .loadBundle(bundleString1) .expectLimboDocs(docA.key) .expectEvents(query, { - added: [doc('collection/a', 500, { key: 'b' })], + added: [doc('collection/a', 500, { value: 'b' })], fromCache: true }) // .watchAcksFull(limboQuery, 1002, docA1) @@ -259,7 +264,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { .watchSnapshots(1002) .expectLimboDocs() .expectEvents(query, { - removed: [doc('collection/a', 500, { key: 'b' })], + removed: [doc('collection/a', 500, { value: 'b' })], fromCache: false }) ); @@ -270,13 +275,13 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { ['multi-client'], () => { const query = Query.atPath(path('collection')); - const docA = doc('collection/a', 250, { key: 'a' }); + const docA = doc('collection/a', 250, { value: 'a' }); const bundleString1 = bundleWithDocument({ key: docA.key, readTime: 500, createTime: 250, updateTime: 500, - content: { key: 'b' } + content: { value: 'b' } }); return client(0) @@ -292,7 +297,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { // TODO(wuandy): Loading from secondary client does not notify other // clients for now. We need to fix it and uncomment below. // .expectEvents(query, { - // modified: [doc('collection/a', 500, { key: 'b' })], + // modified: [doc('collection/a', 500, { value: 'b' })], // }) } ); @@ -302,13 +307,13 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { ['multi-client'], () => { const query = Query.atPath(path('collection')); - const docA = doc('collection/a', 250, { key: 'a' }); + const docA = doc('collection/a', 250, { value: 'a' }); const bundleString1 = bundleWithDocument({ key: docA.key, readTime: 500, createTime: 250, updateTime: 500, - content: { key: 'b' } + content: { value: 'b' } }); return client(0) @@ -324,7 +329,7 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { }) .loadBundle(bundleString1) .expectEvents(query, { - modified: [doc('collection/a', 500, { key: 'b' })] + modified: [doc('collection/a', 500, { value: 'b' })] }); } ); @@ -334,13 +339,13 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { ['multi-client'], () => { const query = Query.atPath(path('collection')); - const docA = doc('collection/a', 250, { key: 'a' }); + const docA = doc('collection/a', 250, { value: 'a' }); const bundleString1 = bundleWithDocument({ key: docA.key, readTime: 500, createTime: 250, updateTime: 500, - content: { key: 'b' } + content: { value: 'b' } }); return client(0) @@ -357,11 +362,11 @@ describeSpec('Bundles:', ['no-ios', 'no-android'], () => { .client(0) .loadBundle(bundleString1) .expectEvents(query, { - modified: [doc('collection/a', 500, { key: 'b' })] + modified: [doc('collection/a', 500, { value: 'b' })] }) .client(1) .expectEvents(query, { - modified: [doc('collection/a', 500, { key: 'b' })] + modified: [doc('collection/a', 500, { value: 'b' })] }); } ); From 609415301ef596b94810f341073890313469f49d Mon Sep 17 00:00:00 2001 From: Wu-Hui Date: Fri, 10 Jul 2020 20:44:36 -0400 Subject: [PATCH 39/39] Even more feedbacks. --- packages/firestore/src/core/bundle.ts | 2 +- packages/firestore/src/core/sync_engine.ts | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index 6e4bbfb0efe..55f00ffb39b 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -143,7 +143,7 @@ export function bundleSuccessProgress( export class BundleLoadResult { constructor( readonly progress: firestore.LoadBundleTaskProgress, - readonly changedDocs?: MaybeDocumentMap + readonly changedDocs: MaybeDocumentMap ) {} } diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 603cf2a514a..759af7c2a9b 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -1432,15 +1432,13 @@ async function loadBundleImpl( } const result = await loader.complete(); - if (result.changedDocs) { - // TODO(b/160876443): This currently raises snapshots with - // `fromCache=false` if users already listen to some queries and bundles - // has newer version. - await syncEngine.emitNewSnapsAndNotifyLocalStore( - result.changedDocs, - /* remoteEvent */ undefined - ); - } + // TODO(b/160876443): This currently raises snapshots with + // `fromCache=false` if users already listen to some queries and bundles + // has newer version. + await syncEngine.emitNewSnapsAndNotifyLocalStore( + result.changedDocs, + /* remoteEvent */ undefined + ); // Save metadata, so loading the same bundle will skip. await saveBundle(syncEngine.localStore, metadata);