diff --git a/packages/firestore/exp/src/api/database.ts b/packages/firestore/exp/src/api/database.ts index f9d2f49dd1d..0a83bc05150 100644 --- a/packages/firestore/exp/src/api/database.ts +++ b/packages/firestore/exp/src/api/database.ts @@ -46,7 +46,6 @@ import { indexedDbStoragePrefix } from '../../../src/local/indexeddb_persistence'; import { LoadBundleTask } from '../../../src/api/bundle'; -import { Query } from '../../../lite'; import { getLocalStore, getPersistence, @@ -332,9 +331,17 @@ export function loadBundle( const firestoreImpl = cast(firestore, Firestore); const resultTask = new LoadBundleTask(); // eslint-disable-next-line @typescript-eslint/no-floating-promises - getSyncEngine(firestoreImpl).then(syncEngine => - enqueueLoadBundle(firestoreImpl._queue, syncEngine, bundleData, resultTask) - ); + getSyncEngine(firestoreImpl).then(async syncEngine => { + const databaseId = (await firestoreImpl._getConfiguration()).databaseInfo + .databaseId; + enqueueLoadBundle( + databaseId, + firestoreImpl._queue, + syncEngine, + bundleData, + resultTask + ); + }); return resultTask; } diff --git a/packages/firestore/src/core/bundle.ts b/packages/firestore/src/core/bundle.ts index 55f00ffb39b..8625f4f0c23 100644 --- a/packages/firestore/src/core/bundle.ts +++ b/packages/firestore/src/core/bundle.ts @@ -35,7 +35,11 @@ import { saveNamedQuery } from '../local/local_store'; import { SizedBundleElement } from '../util/bundle_reader'; -import { MaybeDocumentMap } from '../model/collections'; +import { + documentKeySet, + DocumentKeySet, + MaybeDocumentMap +} from '../model/collections'; import { BundleMetadata } from '../protos/firestore_bundle_proto'; /** @@ -79,7 +83,7 @@ export type BundledDocuments = BundledDocument[]; * Helper to convert objects from bundles to model objects in the SDK. */ export class BundleConverter { - constructor(private serializer: JsonProtoSerializer) {} + constructor(private readonly serializer: JsonProtoSerializer) {} toDocumentKey(name: string): DocumentKey { return fromName(this.serializer, name); @@ -161,7 +165,8 @@ export class BundleLoader { constructor( private metadata: bundleProto.BundleMetadata, - private localStore: LocalStore + private localStore: LocalStore, + private serializer: JsonProtoSerializer ) { this.progress = bundleInitialProgress(metadata); } @@ -208,6 +213,28 @@ export class BundleLoader { return null; } + private getQueryDocumentMapping( + documents: BundledDocuments + ): Map { + const queryDocumentMap = new Map(); + const bundleConverter = new BundleConverter(this.serializer); + for (const bundleDoc of documents) { + if (bundleDoc.metadata.queries) { + const documentKey = bundleConverter.toDocumentKey( + bundleDoc.metadata.name! + ); + for (const queryName of bundleDoc.metadata.queries) { + const documentKeys = ( + queryDocumentMap.get(queryName) || documentKeySet() + ).add(documentKey); + queryDocumentMap.set(queryName, documentKeys); + } + } + } + + return queryDocumentMap; + } + /** * Update the progress to 'Success' and return the updated progress. */ @@ -218,16 +245,18 @@ export class BundleLoader { 'Bundled documents ends with a document metadata and missing document.' ); - for (const q of this.queries) { - await saveNamedQuery(this.localStore, q); - } - - const changedDocs = await applyBundleDocuments( + const changedDocuments = await applyBundleDocuments( this.localStore, this.documents ); + const queryDocumentMap = this.getQueryDocumentMapping(this.documents); + + for (const q of this.queries) { + await saveNamedQuery(this.localStore, q, queryDocumentMap.get(q.name!)); + } + this.progress.taskState = 'Success'; - return new BundleLoadResult({ ...this.progress }, changedDocs); + return new BundleLoadResult({ ...this.progress }, changedDocuments); } } diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index 1972d83955c..88ea5e21eda 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -55,9 +55,10 @@ import { TransactionRunner } from './transaction_runner'; import { Datastore } from '../remote/datastore'; import { BundleReader } from '../util/bundle_reader'; import { LoadBundleTask } from '../api/bundle'; -import { newTextEncoder } from '../platform/serializer'; +import { newSerializer, newTextEncoder } from '../platform/serializer'; import { toByteStreamReader } from '../platform/byte_stream_reader'; import { NamedQuery } from './bundle'; +import { JsonProtoSerializer } from '../remote/serializer'; const LOG_TAG = 'FirestoreClient'; export const MAX_CONCURRENT_LIMBO_RESOLUTIONS = 100; @@ -537,7 +538,10 @@ export class FirestoreClient { ): void { this.verifyNotTerminated(); - const reader = createBundleReader(data); + const reader = createBundleReader( + data, + newSerializer(this.databaseInfo.databaseId) + ); this.asyncQueue.enqueueAndForget(async () => { loadBundle(this.syncEngine, reader, resultTask); return resultTask.catch(e => { @@ -798,7 +802,8 @@ export function enqueueExecuteQueryViaSnapshotListener( } function createBundleReader( - data: ReadableStream | ArrayBuffer | string + data: ReadableStream | ArrayBuffer | string, + serializer: JsonProtoSerializer ): BundleReader { let content: ReadableStream | ArrayBuffer; if (typeof data === 'string') { @@ -806,16 +811,17 @@ function createBundleReader( } else { content = data; } - return new BundleReader(toByteStreamReader(content)); + return new BundleReader(toByteStreamReader(content), serializer); } export function enqueueLoadBundle( + databaseId: DatabaseId, asyncQueue: AsyncQueue, syncEngine: SyncEngine, data: ReadableStream | ArrayBuffer | string, resultTask: LoadBundleTask ): void { - const reader = createBundleReader(data); + const reader = createBundleReader(data, newSerializer(databaseId)); asyncQueue.enqueueAndForget(async () => { loadBundle(syncEngine, reader, resultTask); return resultTask.catch(e => { diff --git a/packages/firestore/src/core/sync_engine.ts b/packages/firestore/src/core/sync_engine.ts index 2dd5c81abe7..ae4fd542d64 100644 --- a/packages/firestore/src/core/sync_engine.ts +++ b/packages/firestore/src/core/sync_engine.ts @@ -1358,7 +1358,11 @@ async function loadBundleImpl( task._updateProgress(bundleInitialProgress(metadata)); - const loader = new BundleLoader(metadata, syncEngine.localStore); + const loader = new BundleLoader( + metadata, + syncEngine.localStore, + reader.serializer + ); let element = await reader.nextElement(); while (element) { debugAssert( diff --git a/packages/firestore/src/local/local_store.ts b/packages/firestore/src/local/local_store.ts index cfb6ee8370e..94c87e852aa 100644 --- a/packages/firestore/src/local/local_store.ts +++ b/packages/firestore/src/local/local_store.ts @@ -1373,7 +1373,8 @@ export function getNamedQuery( */ export async function saveNamedQuery( localStore: LocalStore, - query: bundleProto.NamedQuery + query: bundleProto.NamedQuery, + documents: DocumentKeySet = documentKeySet() ): Promise { // Allocate a target for the named query such that it can be resumed // from associated read time if users use it to listen. @@ -1383,27 +1384,46 @@ export async function saveNamedQuery( const allocated = await localStore.allocateTarget( queryToTarget(fromBundledQuery(query.bundledQuery!)) ); + const localStoreImpl = debugCast(localStore, LocalStoreImpl); return localStoreImpl.persistence.runTransaction( 'Save named query', 'readwrite', transaction => { - // Update allocated target's read time, if the bundle's read time is newer. - let updateReadTime = PersistencePromise.resolve(); const readTime = fromVersion(query.readTime!); - if (allocated.snapshotVersion.compareTo(readTime) < 0) { - const newTargetData = allocated.withResumeToken( - ByteString.EMPTY_BYTE_STRING, - readTime - ); - updateReadTime = localStoreImpl.targetCache.updateTargetData( - transaction, - newTargetData - ); + // Simply save the query itself if it is older than what the SDK already + // has. + if (allocated.snapshotVersion.compareTo(readTime) >= 0) { + return localStoreImpl.bundleCache.saveNamedQuery(transaction, query); } - return updateReadTime.next(() => - localStoreImpl.bundleCache.saveNamedQuery(transaction, query) + + // Update existing target data because the query from the bundle is newer. + const newTargetData = allocated.withResumeToken( + ByteString.EMPTY_BYTE_STRING, + readTime ); + localStoreImpl.targetDataByTarget = localStoreImpl.targetDataByTarget.insert( + newTargetData.targetId, + newTargetData + ); + return localStoreImpl.targetCache + .updateTargetData(transaction, newTargetData) + .next(() => + localStoreImpl.targetCache.removeMatchingKeysForTargetId( + transaction, + allocated.targetId + ) + ) + .next(() => + localStoreImpl.targetCache.addMatchingKeys( + transaction, + documents, + allocated.targetId + ) + ) + .next(() => + localStoreImpl.bundleCache.saveNamedQuery(transaction, query) + ); } ); } diff --git a/packages/firestore/src/protos/firestore/bundle.proto b/packages/firestore/src/protos/firestore/bundle.proto index ca19071e71f..ee7954e664c 100644 --- a/packages/firestore/src/protos/firestore/bundle.proto +++ b/packages/firestore/src/protos/firestore/bundle.proto @@ -79,6 +79,9 @@ message BundledDocumentMetadata { // Whether the document exists. bool exists = 3; + + // The names of the queries in this bundle that this document matches to. + repeated string queries = 4; } // Metadata describing the bundle file/stream. diff --git a/packages/firestore/src/protos/firestore_bundle_proto.ts b/packages/firestore/src/protos/firestore_bundle_proto.ts index 49b8ef07c36..304fe3e68c7 100644 --- a/packages/firestore/src/protos/firestore_bundle_proto.ts +++ b/packages/firestore/src/protos/firestore_bundle_proto.ts @@ -54,6 +54,9 @@ export interface BundledDocumentMetadata { /** BundledDocumentMetadata exists */ exists?: boolean | null; + + /** The names of the queries in this bundle that this document matches to. */ + queries?: string[]; } /** Properties of a BundleMetadata. */ diff --git a/packages/firestore/src/util/bundle_reader.ts b/packages/firestore/src/util/bundle_reader.ts index d2e9c23ac58..ae3badda147 100644 --- a/packages/firestore/src/util/bundle_reader.ts +++ b/packages/firestore/src/util/bundle_reader.ts @@ -23,6 +23,7 @@ import { Deferred } from './promise'; import { debugAssert } from './assert'; import { toByteStreamReader } from '../platform/byte_stream_reader'; import { newTextDecoder } from '../platform/serializer'; +import { JsonProtoSerializer } from '../remote/serializer'; /** * A complete element in the bundle stream, together with the byte length it @@ -70,13 +71,20 @@ export class BundleReader { /** The decoder used to parse binary data into strings. */ private textDecoder: TextDecoder; - static fromBundleSource(source: BundleSource): BundleReader { - return new BundleReader(toByteStreamReader(source, BYTES_PER_READ)); + static fromBundleSource( + source: BundleSource, + serializer: JsonProtoSerializer + ): BundleReader { + return new BundleReader( + toByteStreamReader(source, BYTES_PER_READ), + serializer + ); } constructor( /** The reader to read from underlying binary bundle data source. */ - private reader: ReadableStreamReader + private reader: ReadableStreamReader, + readonly serializer: JsonProtoSerializer ) { this.textDecoder = newTextDecoder(); // Read the metadata (which is the first element). diff --git a/packages/firestore/test/unit/local/local_store.test.ts b/packages/firestore/test/unit/local/local_store.test.ts index 9e3871a9af6..7ec66099511 100644 --- a/packages/firestore/test/unit/local/local_store.test.ts +++ b/packages/firestore/test/unit/local/local_store.test.ts @@ -48,6 +48,7 @@ import { LocalViewChanges } from '../../../src/local/local_view_changes'; import { Persistence } from '../../../src/local/persistence'; import { SimpleQueryEngine } from '../../../src/local/simple_query_engine'; import { + DocumentKeySet, documentKeySet, MaybeDocumentMap } from '../../../src/model/collections'; @@ -91,6 +92,7 @@ import { query, setMutation, TestBundledDocuments, + TestNamedQuery, TestSnapshotVersion, transformMutation, unknownDoc, @@ -137,7 +139,7 @@ class LocalStoreTester { | RemoteEvent | LocalViewChanges | TestBundledDocuments - | bundleProto.NamedQuery + | TestNamedQuery ): LocalStoreTester { if (op instanceof Mutation) { return this.afterMutations([op]); @@ -188,17 +190,21 @@ class LocalStoreTester { this.promiseChain = this.promiseChain .then(() => applyBundleDocuments(this.localStore, documents)) - .then((result: MaybeDocumentMap) => { + .then(result => { this.lastChanges = result; }); return this; } - afterNamedQuery(namedQuery: bundleProto.NamedQuery): LocalStoreTester { + afterNamedQuery(testQuery: TestNamedQuery): LocalStoreTester { this.prepareNextStep(); this.promiseChain = this.promiseChain.then(() => - saveNamedQuery(this.localStore, namedQuery) + saveNamedQuery( + this.localStore, + testQuery.namedQuery, + testQuery.matchingDocuments + ) ); return this; } @@ -432,6 +438,29 @@ class LocalStoreTester { return this; } + toHaveQueryDocumentMapping( + persistence: Persistence, + targetId: TargetId, + expectedKeys: DocumentKeySet + ): LocalStoreTester { + this.promiseChain = this.promiseChain.then(() => { + return persistence.runTransaction( + 'toHaveQueryDocumentMapping', + 'readonly', + transaction => { + return persistence + .getTargetCache() + .getMatchingKeysForTargetId(transaction, targetId) + .next(matchedKeys => { + expect(matchedKeys.isEqual(expectedKeys)).to.be.true; + }); + } + ); + }); + + return this; + } + toHaveNewerBundle( metadata: bundleProto.BundleMetadata, expected: boolean @@ -1706,6 +1735,68 @@ function genericLocalStoreTests( .finish(); }); + it('loading named queries allocates targets and updates target document mapping', async () => { + const expectedQueryDocumentMap = new Map([ + ['query-1', documentKeySet(key('foo1/bar'))], + ['query-2', documentKeySet(key('foo2/bar'))] + ]); + const version1 = SnapshotVersion.fromTimestamp(Timestamp.fromMillis(10000)); + const version2 = SnapshotVersion.fromTimestamp(Timestamp.fromMillis(20000)); + + return expectLocalStore() + .after( + bundledDocuments( + [doc('foo1/bar', 1, { sum: 1337 }), doc('foo2/bar', 2, { sum: 42 })], + [['query-1'], ['query-2']] + ) + ) + .toReturnChanged( + doc('foo1/bar', 1, { sum: 1337 }), + doc('foo2/bar', 2, { sum: 42 }) + ) + .toContain(doc('foo1/bar', 1, { sum: 1337 })) + .toContain(doc('foo2/bar', 2, { sum: 42 })) + .after( + namedQuery( + 'query-1', + query('foo1'), + /* limitType */ 'FIRST', + version1, + expectedQueryDocumentMap.get('query-1') + ) + ) + .toHaveNamedQuery({ + name: 'query-1', + query: query('foo1'), + readTime: version1 + }) + .toHaveQueryDocumentMapping( + persistence, + /*targetId*/ 2, + /*expectedKeys*/ documentKeySet(key('foo1/bar')) + ) + .after( + namedQuery( + 'query-2', + query('foo2'), + /* limitType */ 'FIRST', + version2, + expectedQueryDocumentMap.get('query-2') + ) + ) + .toHaveNamedQuery({ + name: 'query-2', + query: query('foo2'), + readTime: version2 + }) + .toHaveQueryDocumentMapping( + persistence, + /*targetId*/ 4, + /*expectedKeys*/ documentKeySet(key('foo2/bar')) + ) + .finish(); + }); + it('handles saving and loading limit to last queries', async () => { const now = Timestamp.now(); return expectLocalStore() diff --git a/packages/firestore/test/unit/specs/spec_test_runner.ts b/packages/firestore/test/unit/specs/spec_test_runner.ts index 20968d29501..7443b673ea0 100644 --- a/packages/firestore/test/unit/specs/spec_test_runner.ts +++ b/packages/firestore/test/unit/specs/spec_test_runner.ts @@ -477,7 +477,8 @@ abstract class TestRunner { private async doLoadBundle(bundle: string): Promise { const reader = new BundleReader( - toByteStreamReader(newTextEncoder().encode(bundle)) + toByteStreamReader(newTextEncoder().encode(bundle)), + this.serializer ); const task = new LoadBundleTask(); return this.queue.enqueue(async () => { diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index 6b74cb02577..a0a6151c36d 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -40,6 +40,7 @@ import { doc2 } from './bundle_data'; import { newTextEncoder } from '../../../src/platform/serializer'; +import { JSON_SERIALIZER } from '../local/persistence_test_helpers'; use(chaiAsPromised); @@ -92,7 +93,10 @@ describe('Bundle ', () => { function genericBundleReadingTests(bytesPerRead: number): void { function bundleFromString(s: string): BundleReader { - return new BundleReader(byteStreamReaderFromString(s, bytesPerRead)); + return new BundleReader( + byteStreamReaderFromString(s, bytesPerRead), + JSON_SERIALIZER + ); } async function getAllElements( diff --git a/packages/firestore/test/util/helpers.ts b/packages/firestore/test/util/helpers.ts index fa358ab76f7..ec3e10cc0b4 100644 --- a/packages/firestore/test/util/helpers.ts +++ b/packages/firestore/test/util/helpers.ts @@ -445,14 +445,16 @@ export class TestBundledDocuments { } export function bundledDocuments( - documents: MaybeDocument[] + documents: MaybeDocument[], + queryNames?: string[][] ): TestBundledDocuments { - const result = documents.map(d => { + const result = documents.map((d, index) => { return { metadata: { name: toName(JSON_SERIALIZER, d.key), readTime: toVersion(JSON_SERIALIZER, d.version), - exists: d instanceof Document + exists: d instanceof Document, + queries: queryNames ? queryNames[index] : undefined }, document: d instanceof Document ? toDocument(JSON_SERIALIZER, d) : undefined @@ -462,21 +464,32 @@ export function bundledDocuments( return new TestBundledDocuments(result); } +export class TestNamedQuery { + constructor( + public namedQuery: bundleProto.NamedQuery, + public matchingDocuments: DocumentKeySet + ) {} +} + export function namedQuery( name: string, query: Query, limitType: bundleProto.LimitType, - readTime: SnapshotVersion -): bundleProto.NamedQuery { + readTime: SnapshotVersion, + matchingDocuments: DocumentKeySet = documentKeySet() +): TestNamedQuery { return { - name, - readTime: toTimestamp(JSON_SERIALIZER, readTime.toTimestamp()), - bundledQuery: { - parent: toQueryTarget(JSON_SERIALIZER, queryToTarget(query)).parent, - limitType, - structuredQuery: toQueryTarget(JSON_SERIALIZER, queryToTarget(query)) - .structuredQuery - } + namedQuery: { + name, + readTime: toTimestamp(JSON_SERIALIZER, readTime.toTimestamp()), + bundledQuery: { + parent: toQueryTarget(JSON_SERIALIZER, queryToTarget(query)).parent, + limitType, + structuredQuery: toQueryTarget(JSON_SERIALIZER, queryToTarget(query)) + .structuredQuery + } + }, + matchingDocuments }; }