Firestore: QueryTest.java: improve the test that resumes a query with existence filter to actually validate the existence filter.

firebase-firestore/src/androidTest/java/com/google/firebase/firestore/QueryTest.java

@@ -37,6 +37,7 @@
 import com.google.android.gms.tasks.Task;
 import com.google.common.collect.Lists;
 import com.google.firebase.firestore.Query.Direction;
+import com.google.firebase.firestore.remote.ExistenceFilterMismatchListener;
 import com.google.firebase.firestore.testutil.EventAccumulator;
 import com.google.firebase.firestore.testutil.IntegrationTestUtil;
 import java.util.ArrayList;
@@ -1053,6 +1054,7 @@ public void resumingAQueryShouldUseExistenceFilterToDetectDeletes() throws Excep
         createdDocuments.add(documentSnapshot.getReference());
       }
     }
+    assertWithMessage("createdDocuments").that(createdDocuments).hasSize(100);
 
     // Delete 50 of the 100 documents. Do this in a transaction, rather than
     // DocumentReference.delete(), to avoid affecting the local cache.
@@ -1069,13 +1071,33 @@ public void resumingAQueryShouldUseExistenceFilterToDetectDeletes() throws Excep
                   }
                   return null;
                 }));
+    assertWithMessage("deletedDocumentIds").that(deletedDocumentIds).hasSize(50);
 
     // Wait for 10 seconds, during which Watch will stop tracking the query and will send an
     // existence filter rather than "delete" events when the query is resumed.
     Thread.sleep(10000);
 
-    // Resume the query and save the resulting snapshot for verification.
-    QuerySnapshot snapshot2 = waitFor(collection.get());
+    // Resume the query and save the resulting snapshot for verification. Use some internal testing
+    // hooks to "capture" the existence filter mismatches to verify them.
+    ExistenceFilterMismatchListener existenceFilterMismatchListener =
+        new ExistenceFilterMismatchListener();
+    QuerySnapshot snapshot2;
+    ExistenceFilterMismatchListener.ExistenceFilterMismatchInfo existenceFilterMismatchInfo;
+    try {
+      existenceFilterMismatchListener.startListening();
+      snapshot2 = waitFor(collection.get());
+      // TODO(b/270731363): Remove the "if" condition below once the Firestore Emulator is fixed
+      //  to send an existence filter.
+      if (isRunningAgainstEmulator()) {
+        existenceFilterMismatchInfo = null;
+      } else {
+        existenceFilterMismatchInfo =
+            existenceFilterMismatchListener.getOrWaitForExistenceFilterMismatch(
+                /*timeoutMillis=*/ 5000);
+      }
+    } finally {
+      existenceFilterMismatchListener.stopListening();
+    }
 
     // Verify that the snapshot from the resumed query contains the expected documents; that is,
     // that it contains the 50 documents that were _not_ deleted.
@@ -1098,6 +1120,26 @@ public void resumingAQueryShouldUseExistenceFilterToDetectDeletes() throws Excep
           .that(actualDocumentIds)
           .containsExactlyElementsIn(expectedDocumentIds);
     }
+
+    // Skip the verification of the existence filter mismatch when testing against the Firestore
+    // emulator because the Firestore emulator fails to to send an existence filter at all.
+    // TODO(b/270731363): Enable the verification of the existence filter mismatch once the
+    // Firestore emulator is fixed to send an existence filter.
+    if (isRunningAgainstEmulator()) {
+      return;
+    }
+
+    // Verify that Watch sent an existence filter with the correct counts when the query was
+    // resumed.
+    assertWithMessage("Watch should have sent an existence filter")
+        .that(existenceFilterMismatchInfo)
+        .isNotNull();
+    assertWithMessage("localCacheCount")
+        .that(existenceFilterMismatchInfo.localCacheCount())
+        .isEqualTo(100);
+    assertWithMessage("existenceFilterCount")
+        .that(existenceFilterMismatchInfo.existenceFilterCount())
+        .isEqualTo(50);
   }
 
   @Test

firebase-firestore/src/androidTest/java/com/google/firebase/firestore/remote/ExistenceFilterMismatchListener.java

@@ -0,0 +1,150 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.firebase.firestore.remote;
+
+import android.os.SystemClock;
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+import com.google.firebase.firestore.ListenerRegistration;
+import java.util.ArrayList;
+
+/**
+ * Provides a mechanism for tests to listen for existence filter mismatches in the Watch "listen"
+ * stream.
+ */
+public final class ExistenceFilterMismatchListener {
+
+  private TestingHooksExistenceFilterMismatchListenerImpl listener;
+  private ListenerRegistration listenerRegistration;
+
+  /**
+   * Starts listening for existence filter mismatches.
+   *
+   * @throws IllegalStateException if this object is already started.
+   * @see #stopListening
+   */
+  public synchronized void startListening() {
+    if (listener != null) {
+      throw new IllegalStateException("already registered");
+    }
+    listener = new TestingHooksExistenceFilterMismatchListenerImpl();
+    listenerRegistration = TestingHooks.getInstance().addExistenceFilterMismatchListener(listener);
+  }
+
+  /**
+   * Stops listening for existence filter mismatches.
+   *
+   * <p>If listening has not been started then this method does nothing.
+   *
+   * @see #startListening
+   */
+  public synchronized void stopListening() {
+    if (listenerRegistration != null) {
+      listenerRegistration.remove();
+    }
+    listenerRegistration = null;
+    listener = null;
+  }
+
+  /**
+   * Returns the oldest existence filter mismatch observed, waiting if none has yet been observed.
+   *
+   * <p>The oldest existence filter mismatch observed since the most recent successful invocation of
+   * {@link #startListening} will be returned. A subsequent invocation of this method will return
+   * the second-oldest existence filter mismatch observed, and so on. An invocation of {@link
+   * #stopListening} followed by another invocation of {@link #startListening} will discard any
+   * existence filter mismatches that occurred while previously started and will start observing
+   * afresh.
+   *
+   * @param timeoutMillis the maximum amount of time, in milliseconds, to wait for an existence
+   *     filter mismatch to occur.
+   * @return information about the existence filter mismatch that occurred.
+   * @throws InterruptedException if waiting is interrupted.
+   * @throws IllegalStateException if this object has not been started by {@link #startListening}.
+   * @throws IllegalArgumentException if the given timeout is less than or equal to zero.
+   */
+  @Nullable
+  public ExistenceFilterMismatchInfo getOrWaitForExistenceFilterMismatch(long timeoutMillis)
+      throws InterruptedException {
+    if (timeoutMillis <= 0) {
+      throw new IllegalArgumentException("invalid timeout: " + timeoutMillis);
+    }
+
+    TestingHooksExistenceFilterMismatchListenerImpl registeredListener;
+    synchronized (this) {
+      registeredListener = listener;
+    }
+
+    if (registeredListener == null) {
+      throw new IllegalStateException(
+          "must be registered before waiting for an existence filter mismatch");
+    }
+
+    return registeredListener.getOrWaitForExistenceFilterMismatch(timeoutMillis);
+  }
+
+  private static final class TestingHooksExistenceFilterMismatchListenerImpl
+      implements TestingHooks.ExistenceFilterMismatchListener {
+
+    private final ArrayList<ExistenceFilterMismatchInfo> existenceFilterMismatches =
+        new ArrayList<>();
+
+    @Override
+    public synchronized void onExistenceFilterMismatch(
+        @NonNull TestingHooks.ExistenceFilterMismatchInfo info) {
+      existenceFilterMismatches.add(new ExistenceFilterMismatchInfo(info));
+      notifyAll();
+    }
+
+    @Nullable
+    synchronized ExistenceFilterMismatchInfo getOrWaitForExistenceFilterMismatch(long timeoutMillis)
+        throws InterruptedException {
+      if (timeoutMillis <= 0) {
+        throw new IllegalArgumentException("invalid timeout: " + timeoutMillis);
+      }
+
+      long endTimeMillis = SystemClock.uptimeMillis() + timeoutMillis;
+      while (true) {
+        if (existenceFilterMismatches.size() > 0) {
+          return existenceFilterMismatches.remove(0);
+        }
+        long currentWaitMillis = endTimeMillis - SystemClock.uptimeMillis();
+        if (currentWaitMillis <= 0) {
+          return null;
+        }
+
+        wait(currentWaitMillis);
+      }
+    }
+  }
+
+  /** @see TestingHooks.ExistenceFilterMismatchInfo */
+  public static final class ExistenceFilterMismatchInfo {
+
+    private final TestingHooks.ExistenceFilterMismatchInfo info;
+
+    ExistenceFilterMismatchInfo(@NonNull TestingHooks.ExistenceFilterMismatchInfo info) {
+      this.info = info;
+    }
+
+    public int localCacheCount() {
+      return info.localCacheCount();
+    }
+
+    public int existenceFilterCount() {
+      return info.existenceFilterCount();
+    }
+  }
+}

firebase-firestore/src/main/java/com/google/firebase/firestore/remote/TestingHooks.java

@@ -0,0 +1,150 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.firebase.firestore.remote;
+
+import static com.google.firebase.firestore.util.Preconditions.checkNotNull;
+
+import androidx.annotation.AnyThread;
+import androidx.annotation.NonNull;
+import androidx.annotation.VisibleForTesting;
+import com.google.auto.value.AutoValue;
+import com.google.firebase.firestore.ListenerRegistration;
+import com.google.firebase.firestore.util.Executors;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Manages "testing hooks", hooks into the internals of the SDK to verify internal state and events
+ * during integration tests.
+ *
+ * <p>Do not use this class except for testing purposes.
+ */
+@VisibleForTesting
+final class TestingHooks {
+
+  private static final TestingHooks instance = new TestingHooks();
+
+  // Use CopyOnWriteArrayList to store the listeners so that we don't need to worry about
+  // synchronizing adds, removes, and traversals.
+  private final CopyOnWriteArrayList<AtomicReference<ExistenceFilterMismatchListener>>
+      existenceFilterMismatchListeners = new CopyOnWriteArrayList<>();
+
+  private TestingHooks() {}
+
+  /** Returns the singleton instance of this class. */
+  @NonNull
+  static TestingHooks getInstance() {
+    return instance;
+  }
+
+  /**
+   * Asynchronously notifies all registered {@link ExistenceFilterMismatchListener}` listeners
+   * registered via {@link #addExistenceFilterMismatchListener}.
+   *
+   * @param info Information about the existence filter mismatch to deliver to the listeners.
+   */
+  void notifyOnExistenceFilterMismatch(@NonNull ExistenceFilterMismatchInfo info) {
+    for (AtomicReference<ExistenceFilterMismatchListener> listenerRef :
+        existenceFilterMismatchListeners) {
+      Executors.BACKGROUND_EXECUTOR.execute(
+          () -> {
+            ExistenceFilterMismatchListener listener = listenerRef.get();
+            if (listener != null) {
+              listener.onExistenceFilterMismatch(info);
+            }
+          });
+    }
+  }
+
+  /**
+   * Registers a {@link ExistenceFilterMismatchListener} to be notified when an existence filter
+   * mismatch occurs in the Watch listen stream.
+   *
+   * <p>The relative order in which callbacks are notified is unspecified; do not rely on any
+   * particular ordering. If a given callback is registered multiple times then it will be notified
+   * multiple times, once per registration.
+   *
+   * <p>The thread on which the callback occurs is unspecified; listeners should perform their work
+   * as quickly as possible and return to avoid blocking any critical work. In particular, the
+   * listener callbacks should <em>not</em> block or perform long-running operations. Listener
+   * callbacks can occur concurrently with other callbacks on the same and other listeners.
+   *
+   * @param listener the listener to register.
+   * @return an object that unregisters the given listener via its {@link
+   *     ListenerRegistration#remove} method; only the first unregistration request does anything;
+   *     all subsequent requests do nothing.
+   */
+  ListenerRegistration addExistenceFilterMismatchListener(
+      @NonNull ExistenceFilterMismatchListener listener) {
+    checkNotNull(listener, "a null listener is not allowed");
+
+    AtomicReference<ExistenceFilterMismatchListener> listenerRef = new AtomicReference<>(listener);
+    existenceFilterMismatchListeners.add(listenerRef);
+
+    return () -> {
+      listenerRef.set(null);
+      existenceFilterMismatchListeners.remove(listenerRef);
+    };
+  }
+
+  /**
+   * Implementations of this interface can be registered with {@link
+   * #addExistenceFilterMismatchListener}.
+   */
+  interface ExistenceFilterMismatchListener {
+
+    /**
+     * Invoked when an existence filter mismatch occurs.
+     *
+     * @param info information about the existence filter mismatch.
+     */
+    @AnyThread
+    void onExistenceFilterMismatch(@NonNull ExistenceFilterMismatchInfo info);
+  }
+
+  /**
+   * Information about an existence filter mismatch, as specified to listeners registered with
+   * {@link #addExistenceFilterMismatchListener}.
+   */
+  @AutoValue
+  abstract static class ExistenceFilterMismatchInfo {
+
+    /**
+     * Creates and returns a new instance of {@link ExistenceFilterMismatchInfo} with the given
+     * values.
+     */
+    static ExistenceFilterMismatchInfo create(int localCacheCount, int existenceFilterCount) {
+      return new AutoValue_TestingHooks_ExistenceFilterMismatchInfo(
+          localCacheCount, existenceFilterCount);
+    }
+
+    /** Returns the number of documents that matched the query in the local cache. */
+    abstract int localCacheCount();
+
+    /**
+     * Returns the number of documents that matched the query on the server, as specified in the
+     * ExistenceFilter message's `count` field.
+     */
+    abstract int existenceFilterCount();
+
+    /**
+     * Convenience method to create and return a new instance of {@link ExistenceFilterMismatchInfo}
+     * with the values taken from the given arguments.
+     */
+    static ExistenceFilterMismatchInfo from(int localCacheCount, ExistenceFilter existenceFilter) {
+      return create(localCacheCount, existenceFilter.getCount());
+    }
+  }
+}

firebase-firestore/src/main/java/com/google/firebase/firestore/remote/WatchChangeAggregator.java

@@ -202,6 +202,11 @@ public void handleExistenceFilter(ExistenceFilterWatchChange watchChange) {
           // `isFromCache:true`.
           resetTarget(targetId);
           pendingTargetResets.add(targetId);
+
+          TestingHooks.getInstance()
+              .notifyOnExistenceFilterMismatch(
+                  TestingHooks.ExistenceFilterMismatchInfo.from(
+                      (int) currentSize, watchChange.getExistenceFilter()));
         }
       }
     }