Firestore: Optimize local cache sync when resuming a query that had docs deleted

.changeset/swift-eels-change.md

@@ -0,0 +1,7 @@
+---
+'@firebase/webchannel-wrapper': minor
+'@firebase/firestore': minor
+'firebase': minor
+---
+
+Optimize local cache synchronization when resuming a query where documents were deleted on the server.

packages/firestore/package.json

@@ -20,7 +20,7 @@
     "dev": "rollup -c -w",
     "lint": "eslint -c .eslintrc.js '**/*.ts' --ignore-path '../../.gitignore'",
     "lint:fix": "eslint --fix -c .eslintrc.js '**/*.ts' --ignore-path '../../.gitignore'",
-    "prettier": "prettier --write '*.js' '@(lite|src|test)/**/*.ts'",
+    "prettier": "prettier --write '*.js' '@(lite|src|test)/**/*.ts' 'test/unit/remote/bloom_filter_golden_test_data/*.json'",
     "test:lite": "ts-node ./scripts/run-tests.ts --emulator --platform node_lite --main=lite/index.ts 'test/lite/**/*.test.ts'",
     "test:lite:prod": "ts-node ./scripts/run-tests.ts --platform node_lite --main=lite/index.ts 'test/lite/**/*.test.ts'",
     "test:lite:browser": "karma start --single-run --lite",

packages/firestore/src/core/sync_engine_impl.ts

@@ -71,7 +71,6 @@ import { primitiveComparator } from '../util/misc';
 import { ObjectMap } from '../util/obj_map';
 import { Deferred } from '../util/promise';
 import { SortedMap } from '../util/sorted_map';
-import { SortedSet } from '../util/sorted_set';
 import { BATCHID_UNKNOWN } from '../util/types';
 
 import {
@@ -316,9 +315,6 @@ export async function syncEngineListen(
       syncEngineImpl.localStore,
       queryToTarget(query)
     );
-    if (syncEngineImpl.isPrimaryClient) {
-      remoteStoreListen(syncEngineImpl.remoteStore, targetData);
-    }
 
     const status = syncEngineImpl.sharedClientState.addLocalQueryTarget(
       targetData.targetId
@@ -331,6 +327,10 @@ export async function syncEngineListen(
       status === 'current',
       targetData.resumeToken
     );
+
+    if (syncEngineImpl.isPrimaryClient) {
+      remoteStoreListen(syncEngineImpl.remoteStore, targetData);
+    }
   }
 
   return viewSnapshot;
@@ -638,7 +638,9 @@ export async function syncEngineRejectListen(
     const event = new RemoteEvent(
       SnapshotVersion.min(),
       /* targetChanges= */ new Map<TargetId, TargetChange>(),
-      /* targetMismatches= */ new SortedSet<TargetId>(primitiveComparator),
+      /* targetMismatches= */ new SortedMap<TargetId, TargetPurpose>(
+        primitiveComparator
+      ),
       documentUpdates,
       resolvedLimboDocuments
     );

packages/firestore/src/local/local_store_impl.ts

@@ -601,7 +601,7 @@ export function localStoreApplyRemoteEventToLocalCache(
         let newTargetData = oldTargetData.withSequenceNumber(
           txn.currentSequenceNumber
         );
-        if (remoteEvent.targetMismatches.has(targetId)) {
+        if (remoteEvent.targetMismatches.get(targetId) !== null) {
           newTargetData = newTargetData
             .withResumeToken(
               ByteString.EMPTY_BYTE_STRING,

packages/firestore/src/local/target_data.ts

@@ -26,10 +26,17 @@ export const enum TargetPurpose {
   Listen,
 
   /**
-   * The query target was used to refill a query after an existence filter mismatch.
+   * The query target was used to refill a query after an existence filter
+   * mismatch.
    */
   ExistenceFilterMismatch,
 
+  /**
+   * The query target was used if the query is the result of a false positive in
+   * the bloom filter.
+   */
+  ExistenceFilterMismatchBloom,
+
   /** The query target was used to resolve a limbo document. */
   LimboResolution
 }
@@ -66,7 +73,13 @@ export class TargetData {
      * matches the target. The resume token essentially identifies a point in
      * time from which the server should resume sending results.
      */
-    readonly resumeToken: ByteString = ByteString.EMPTY_BYTE_STRING
+    readonly resumeToken: ByteString = ByteString.EMPTY_BYTE_STRING,
+    /**
+     * The number of documents that last matched the query at the resume token or
+     * read time. Documents are counted only when making a listen request with
+     * resume token or read time, otherwise, keep it null.
+     */
+    readonly expectedCount: number | null = null
   ) {}
 
   /** Creates a new target data instance with an updated sequence number. */
@@ -78,7 +91,8 @@ export class TargetData {
       sequenceNumber,
       this.snapshotVersion,
       this.lastLimboFreeSnapshotVersion,
-      this.resumeToken
+      this.resumeToken,
+      this.expectedCount
     );
   }
 
@@ -97,7 +111,24 @@ export class TargetData {
       this.sequenceNumber,
       snapshotVersion,
       this.lastLimboFreeSnapshotVersion,
-      resumeToken
+      resumeToken,
+      /* expectedCount= */ null
+    );
+  }
+
+  /**
+   * Creates a new target data instance with an updated expected count.
+   */
+  withExpectedCount(expectedCount: number): TargetData {
+    return new TargetData(
+      this.target,
+      this.targetId,
+      this.purpose,
+      this.sequenceNumber,
+      this.snapshotVersion,
+      this.lastLimboFreeSnapshotVersion,
+      this.resumeToken,
+      expectedCount
     );
   }
 
@@ -115,7 +146,8 @@ export class TargetData {
       this.sequenceNumber,
       this.snapshotVersion,
       lastLimboFreeSnapshotVersion,
-      this.resumeToken
+      this.resumeToken,
+      this.expectedCount
     );
   }
 }

packages/firestore/src/protos/firestore_proto_api.ts

@@ -224,6 +224,15 @@ export declare namespace firestoreV1ApiClientInterfaces {
   interface ExistenceFilter {
     targetId?: number;
     count?: number;
+    unchangedNames?: BloomFilter;
+  }
+  interface BloomFilter {
+    bits?: BitSequence;
+    hashCount?: number;
+  }
+  interface BitSequence {
+    bitmap?: string | Uint8Array;
+    padding?: number;
   }
   interface FieldFilter {
     field?: FieldReference;
@@ -390,6 +399,7 @@ export declare namespace firestoreV1ApiClientInterfaces {
     readTime?: Timestamp;
     targetId?: number;
     once?: boolean;
+    expectedCount?: number | { value: number };
   }
   interface TargetChange {
     targetChangeType?: TargetChangeTargetChangeType;
@@ -454,6 +464,7 @@ export declare type BeginTransactionRequest =
   firestoreV1ApiClientInterfaces.BeginTransactionRequest;
 export declare type BeginTransactionResponse =
   firestoreV1ApiClientInterfaces.BeginTransactionResponse;
+export declare type BloomFilter = firestoreV1ApiClientInterfaces.BloomFilter;
 export declare type CollectionSelector =
   firestoreV1ApiClientInterfaces.CollectionSelector;
 export declare type CommitRequest =

packages/firestore/src/protos/google/firestore/v1/bloom_filter.proto

@@ -0,0 +1,73 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.firestore.v1;
+
+option csharp_namespace = "Google.Cloud.Firestore.V1";
+option go_package = "google.golang.org/genproto/googleapis/firestore/v1;firestore";
+option java_multiple_files = true;
+option java_outer_classname = "BloomFilterProto";
+option java_package = "com.google.firestore.v1";
+option objc_class_prefix = "GCFS";
+option php_namespace = "Google\\Cloud\\Firestore\\V1";
+option ruby_package = "Google::Cloud::Firestore::V1";
+
+// A sequence of bits, encoded in a byte array.
+//
+// Each byte in the `bitmap` byte array stores 8 bits of the sequence. The only
+// exception is the last byte, which may store 8 _or fewer_ bits. The `padding`
+// defines the number of bits of the last byte to be ignored as "padding". The
+// values of these "padding" bits are unspecified and must be ignored.
+//
+// To retrieve the first bit, bit 0, calculate: (bitmap[0] & 0x01) != 0.
+// To retrieve the second bit, bit 1, calculate: (bitmap[0] & 0x02) != 0.
+// To retrieve the third bit, bit 2, calculate: (bitmap[0] & 0x04) != 0.
+// To retrieve the fourth bit, bit 3, calculate: (bitmap[0] & 0x08) != 0.
+// To retrieve bit n, calculate: (bitmap[n / 8] & (0x01 << (n % 8))) != 0.
+//
+// The "size" of a `BitSequence` (the number of bits it contains) is calculated
+// by this formula: (bitmap.length * 8) - padding.
+message BitSequence {
+  // The bytes that encode the bit sequence.
+  // May have a length of zero.
+  bytes bitmap = 1;
+
+  // The number of bits of the last byte in `bitmap` to ignore as "padding".
+  // If the length of `bitmap` is zero, then this value must be 0.
+  // Otherwise, this value must be between 0 and 7, inclusive.
+  int32 padding = 2;
+}
+
+// A bloom filter (https://en.wikipedia.org/wiki/Bloom_filter).
+//
+// The bloom filter hashes the entries with MD5 and treats the resulting 128-bit
+// hash as 2 distinct 64-bit hash values, interpreted as unsigned integers
+// using 2's complement encoding.
+//
+// These two hash values, named h1 and h2, are then used to compute the
+// `hash_count` hash values using the formula, starting at i=0:
+//
+//   h(i) = h1 + (i * h2)
+//
+// These resulting values are then taken modulo the number of bits in the bloom
+// filter to get the bits of the bloom filter to test for the given entry.
+message BloomFilter {
+  // The bloom filter data.
+  BitSequence bits = 1;
+
+  // The number of hashes used by the algorithm.
+  int32 hash_count = 2;
+}

packages/firestore/src/protos/google/firestore/v1/firestore.proto

@@ -26,6 +26,7 @@ import "google/firestore/v1/query.proto";
 import "google/firestore/v1/write.proto";
 import "google/protobuf/empty.proto";
 import "google/protobuf/timestamp.proto";
+import "google/protobuf/wrappers.proto";
 import "google/rpc/status.proto";
 
 option csharp_namespace = "Google.Cloud.Firestore.V1";
@@ -857,6 +858,15 @@ message Target {
 
   // If the target should be removed once it is current and consistent.
   bool once = 6;
+
+  // The number of documents that last matched the query at the resume token or
+  // read time.
+  //
+  // This value is only relevant when a `resume_type` is provided. This value
+  // being present and greater than zero signals that the client wants
+  // `ExistenceFilter.unchanged_names` to be included in the response.
+  //
+  google.protobuf.Int32Value expected_count = 12;
 }
 
 // Targets being watched have changed.

packages/firestore/src/protos/google/firestore/v1/write.proto

@@ -16,6 +16,7 @@ syntax = "proto3";
 
 package google.firestore.v1;
 
+import "google/firestore/v1/bloom_filter.proto";
 import "google/firestore/v1/common.proto";
 import "google/firestore/v1/document.proto";
 import "google/protobuf/timestamp.proto";
@@ -261,4 +262,18 @@ message ExistenceFilter {
   // If different from the count of documents in the client that match, the
   // client must manually determine which documents no longer match the target.
   int32 count = 2;
+
+  // A bloom filter that contains the UTF-8 byte encodings of the resource names
+  // of the documents that match [target_id][google.firestore.v1.ExistenceFilter.target_id], in the
+  // form `projects/{project_id}/databases/{database_id}/documents/{document_path}`
+  // that have NOT changed since the query results indicated by the resume token
+  // or timestamp given in `Target.resume_type`.
+  //
+  // This bloom filter may be omitted at the server's discretion, such as if it
+  // is deemed that the client will not make use of it or if it is too
+  // computationally expensive to calculate or transmit. Clients must gracefully
+  // handle this field being absent by falling back to the logic used before
+  // this field existed; that is, re-add the target without a resume token to
+  // figure out which documents in the client's cache are out of sync.
+  BloomFilter unchanged_names = 3;
 }

packages/firestore/src/protos/protos.json

@@ -928,6 +928,30 @@
                     }
                   }
                 },
+                "BitSequence": {
+                  "fields": {
+                    "bitmap": {
+                      "type": "bytes",
+                      "id": 1
+                    },
+                    "padding": {
+                      "type": "int32",
+                      "id": 2
+                    }
+                  }
+                },
+                "BloomFilter": {
+                  "fields": {
+                    "bits": {
+                      "type": "BitSequence",
+                      "id": 1
+                    },
+                    "hashCount": {
+                      "type": "int32",
+                      "id": 2
+                    }
+                  }
+                },
                 "DocumentMask": {
                   "fields": {
                     "fieldPaths": {
@@ -2052,6 +2076,10 @@
                     "once": {
                       "type": "bool",
                       "id": 6
+                    },
+                    "expectedCount": {
+                      "type": "google.protobuf.Int32Value",
+                      "id": 12
                     }
                   },
                   "nested": {
@@ -2660,6 +2688,10 @@
                     "count": {
                       "type": "int32",
                       "id": 2
+                    },
+                    "unchangedNames": {
+                      "type": "BloomFilter",
+                      "id": 3
                     }
                   }
                 }