Index-Free Queries (code, feature still disabled)

firebase-firestore/ktx/src/test/java/com/google/firebase/firestore/TestUtil.java

@@ -117,7 +117,8 @@ public static QuerySnapshot querySnapshot(
             documentChanges,
             isFromCache,
             mutatedKeys,
-            true,
+            /* synced= */ false,
+            /* didSyncStateChange= */ true,
             /* excludesMetadataChanges= */ false);
     return new QuerySnapshot(query(path), viewSnapshot, FIRESTORE);
   }

firebase-firestore/src/androidTest/java/com/google/firebase/firestore/remote/RemoteStoreTest.java

@@ -24,6 +24,7 @@
 import com.google.firebase.firestore.local.LocalStore;
 import com.google.firebase.firestore.local.MemoryPersistence;
 import com.google.firebase.firestore.local.Persistence;
+import com.google.firebase.firestore.local.SimpleQueryEngine;
 import com.google.firebase.firestore.model.DocumentKey;
 import com.google.firebase.firestore.model.mutation.MutationBatchResult;
 import com.google.firebase.firestore.testutil.IntegrationTestUtil;
@@ -72,9 +73,10 @@ public ImmutableSortedSet<DocumentKey> getRemoteKeysForTarget(int targetId) {
         };
 
     FakeConnectivityMonitor connectivityMonitor = new FakeConnectivityMonitor();
+    SimpleQueryEngine queryEngine = new SimpleQueryEngine();
     Persistence persistence = MemoryPersistence.createEagerGcMemoryPersistence();
     persistence.start();
-    LocalStore localStore = new LocalStore(persistence, User.UNAUTHENTICATED);
+    LocalStore localStore = new LocalStore(persistence, queryEngine, User.UNAUTHENTICATED);
     RemoteStore remoteStore =
         new RemoteStore(callback, localStore, datastore, testQueue, connectivityMonitor);
 

firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java

@@ -37,7 +37,9 @@
 import com.google.firebase.firestore.local.LruGarbageCollector;
 import com.google.firebase.firestore.local.MemoryPersistence;
 import com.google.firebase.firestore.local.Persistence;
+import com.google.firebase.firestore.local.QueryEngine;
 import com.google.firebase.firestore.local.SQLitePersistence;
+import com.google.firebase.firestore.local.SimpleQueryEngine;
 import com.google.firebase.firestore.model.Document;
 import com.google.firebase.firestore.model.DocumentKey;
 import com.google.firebase.firestore.model.MaybeDocument;
@@ -266,7 +268,9 @@ private void initialize(Context context, User user, boolean usePersistence, long
     }
 
     persistence.start();
-    localStore = new LocalStore(persistence, user);
+    // TODO(index-free): Use IndexFreeQueryEngine/IndexedQueryEngine as appropriate.
+    QueryEngine queryEngine = new SimpleQueryEngine();
+    localStore = new LocalStore(persistence, queryEngine, user);
     if (gc != null) {
       lruScheduler = gc.newScheduler(asyncQueue, localStore);
       lruScheduler.start();

firebase-firestore/src/main/java/com/google/firebase/firestore/core/Query.java

@@ -117,6 +117,18 @@ public boolean isCollectionGroupQuery() {
     return collectionGroup != null;
   }
 
+  /**
+   * Returns true if this query does not specify any query constraints that could remove results.
+   */
+  public boolean matchesAllDocuments() {
+    return filters.isEmpty()
+        && limit == NO_LIMIT
+        && startAt == null
+        && endAt == null
+        && (getExplicitOrderBy().isEmpty()
+            || (getExplicitOrderBy().size() == 1 && getFirstOrderByField().isKeyField()));
+  }
+
   /** The filters on the documents returned by the query. */
   public List<Filter> getFilters() {
     return filters;

firebase-firestore/src/main/java/com/google/firebase/firestore/core/QueryListener.java

@@ -80,6 +80,7 @@ public void onViewSnapshot(ViewSnapshot newSnapshot) {
               documentChanges,
               newSnapshot.isFromCache(),
               newSnapshot.getMutatedKeys(),
+              newSnapshot.isSynced(),
               newSnapshot.didSyncStateChange(),
               /* excludesMetadataChanges= */ true);
     }
@@ -158,6 +159,7 @@ private void raiseInitialEvent(ViewSnapshot snapshot) {
             snapshot.getDocuments(),
             snapshot.getMutatedKeys(),
             snapshot.isFromCache(),
+            snapshot.isSynced(),
             snapshot.excludesMetadataChanges());
     raisedInitialEvent = true;
     listener.onEvent(snapshot, null);

firebase-firestore/src/main/java/com/google/firebase/firestore/core/SyncEngine.java

@@ -191,9 +191,10 @@ public int listen(Query query) {
   private ViewSnapshot initializeViewAndComputeSnapshot(QueryData queryData) {
     Query query = queryData.getQuery();
 
-    ImmutableSortedMap<DocumentKey, Document> docs = localStore.executeQuery(query);
     ImmutableSortedSet<DocumentKey> remoteKeys =
         localStore.getRemoteDocumentKeys(queryData.getTargetId());
+    ImmutableSortedMap<DocumentKey, Document> docs =
+        localStore.executeQuery(query, queryData, remoteKeys);
 
     View view = new View(query, remoteKeys);
     View.DocumentChanges viewDocChanges = view.computeDocChanges(docs);
@@ -530,7 +531,8 @@ private void emitNewSnapsAndNotifyLocalStore(
         // against the local store to make sure we didn't lose any good docs that had been past the
         // limit.
         ImmutableSortedMap<DocumentKey, Document> docs =
-            localStore.executeQuery(queryView.getQuery());
+            localStore.executeQuery(
+                queryView.getQuery(), /* queryData= */ null, DocumentKey.emptyKeySet());
         viewDocChanges = view.computeDocChanges(docs, viewDocChanges);
       }
       TargetChange targetChange =

firebase-firestore/src/main/java/com/google/firebase/firestore/core/View.java

@@ -311,6 +311,7 @@ public ViewChange applyChanges(DocumentChanges docChanges, TargetChange targetCh
               viewChanges,
               fromCache,
               docChanges.mutatedKeys,
+              synced,
               syncStatedChanged,
               /* excludesMetadataChanges= */ false);
     }

firebase-firestore/src/main/java/com/google/firebase/firestore/core/ViewSnapshot.java

@@ -37,6 +37,7 @@ public enum SyncState {
   private final List<DocumentViewChange> changes;
   private final boolean isFromCache;
   private final ImmutableSortedSet<DocumentKey> mutatedKeys;
+  private final boolean synced;
   private final boolean didSyncStateChange;
   private boolean excludesMetadataChanges;
 
@@ -47,6 +48,7 @@ public ViewSnapshot(
       List<DocumentViewChange> changes,
       boolean isFromCache,
       ImmutableSortedSet<DocumentKey> mutatedKeys,
+      boolean synced,
       boolean didSyncStateChange,
       boolean excludesMetadataChanges) {
     this.query = query;
@@ -55,6 +57,7 @@ public ViewSnapshot(
     this.changes = changes;
     this.isFromCache = isFromCache;
     this.mutatedKeys = mutatedKeys;
+    this.synced = synced;
     this.didSyncStateChange = didSyncStateChange;
     this.excludesMetadataChanges = excludesMetadataChanges;
   }
@@ -65,6 +68,7 @@ public static ViewSnapshot fromInitialDocuments(
       DocumentSet documents,
       ImmutableSortedSet<DocumentKey> mutatedKeys,
       boolean fromCache,
+      boolean synced,
       boolean excludesMetadataChanges) {
     List<DocumentViewChange> viewChanges = new ArrayList<>();
     for (Document doc : documents) {
@@ -77,6 +81,7 @@ public static ViewSnapshot fromInitialDocuments(
         viewChanges,
         fromCache,
         mutatedKeys,
+        synced,
         /* didSyncStateChange= */ true,
         excludesMetadataChanges);
   }
@@ -85,6 +90,10 @@ public Query getQuery() {
     return query;
   }
 
+  public boolean isSynced() {
+    return synced;
+  }
+
   public DocumentSet getDocuments() {
     return documents;
   }
@@ -131,6 +140,9 @@ public final boolean equals(Object o) {
     if (isFromCache != that.isFromCache) {
       return false;
     }
+    if (synced != that.synced) {
+      return false;
+    }
     if (didSyncStateChange != that.didSyncStateChange) {
       return false;
     }
@@ -160,6 +172,7 @@ public int hashCode() {
     result = 31 * result + changes.hashCode();
     result = 31 * result + mutatedKeys.hashCode();
     result = 31 * result + (isFromCache ? 1 : 0);
+    result = 31 * result + (synced ? 1 : 0);
     result = 31 * result + (didSyncStateChange ? 1 : 0);
     result = 31 * result + (excludesMetadataChanges ? 1 : 0);
     return result;
@@ -179,6 +192,8 @@ public String toString() {
         + isFromCache
         + ", mutatedKeys="
         + mutatedKeys.size()
+        + ", synced="
+        + synced
         + ", didSyncStateChange="
         + didSyncStateChange
         + ", excludesMetadataChanges="

firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexFreeQueryEngine.java

@@ -0,0 +1,166 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.firebase.firestore.local;
+
+import static com.google.firebase.firestore.util.Assert.hardAssert;
+
+import androidx.annotation.Nullable;
+import com.google.firebase.database.collection.ImmutableSortedMap;
+import com.google.firebase.database.collection.ImmutableSortedSet;
+import com.google.firebase.firestore.core.Query;
+import com.google.firebase.firestore.model.Document;
+import com.google.firebase.firestore.model.DocumentKey;
+import com.google.firebase.firestore.model.MaybeDocument;
+import com.google.firebase.firestore.model.SnapshotVersion;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * A query engine that takes advantage of the target document mapping in the QueryCache. The
+ * IndexFreeQueryEngine optimizes query execution by only reading the documents previously matched a
+ * query plus any documents that were edited after the query was last listened to.
+ *
+ * <p>There are some cases where Index-Free queries are not guaranteed to produce to the same
+ * results as full collection scans. In these case, the IndexFreeQueryEngine falls back to a full
+ * query processing. These cases are:
+ *
+ * <ol>
+ *   <li>Limit queries where a document that matched the query previously no longer matches the
+ *       query. In this case, we have to scan all local documents since a document that was sent to
+ *       us as part of a different query result may now fall into the limit.
+ *   <li>Limit queries that include edits that occurred after the last remote snapshot (both
+ *       latency-compensated or committed). Even if an edited document continues to match the query,
+ *       an edit may cause a document to sort below another document that is in the local cache.
+ *   <li>Queries where the last snapshot contained Limbo documents. Even though a Limbo document is
+ *       not part of the backend result set, we need to include Limbo documents in local views to
+ *       ensure consistency between different Query views. If there exists a previous query snapshot
+ *       that contained no limbo documents, we can instead use the older snapshot version for
+ *       Index-Free processing.
+ * </ol>
+ */
+public class IndexFreeQueryEngine implements QueryEngine {
+  private LocalDocumentsView localDocumentsView;
+
+  @Override
+  public void setLocalDocumentsView(LocalDocumentsView localDocuments) {
+    this.localDocumentsView = localDocuments;
+  }
+
+  @Override
+  public ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingQuery(
+      Query query, @Nullable QueryData queryData, ImmutableSortedSet<DocumentKey> remoteKeys) {
+    hardAssert(localDocumentsView != null, "setLocalDocumentsView() not called");
+
+    // Queries that match all document don't benefit from using IndexFreeQueries. It is more
+    // efficient to scan all documents in a collection, rather than to perform individual lookups.
+    if (query.matchesAllDocuments()) {
+      return executeFullCollectionScan(query);
+    }
+
+    // Queries that have never seen a snapshot without limbo free documents should also be run as a
+    // full collection scan.
+    if (queryData == null
+        || queryData.getLastLimboFreeSnapshotVersion().equals(SnapshotVersion.NONE)) {
+      return executeFullCollectionScan(query);
+    }
+
+    ImmutableSortedSet<Document> previousResults = getSortedPreviousResults(query, remoteKeys);
+
+    if (query.hasLimit()
+        && needsRefill(previousResults, remoteKeys, queryData.getLastLimboFreeSnapshotVersion())) {
+      return executeFullCollectionScan(query);
+    }
+
+    // Retrieve all results for documents that were updated since the last limbo-document free
+    // remote snapshot.
+    ImmutableSortedMap<DocumentKey, Document> updatedResults =
+        localDocumentsView.getDocumentsMatchingQuery(
+            query, queryData.getLastLimboFreeSnapshotVersion());
+    for (Document result : previousResults) {
+      updatedResults = updatedResults.insert(result.getKey(), result);
+    }
+
+    return updatedResults;
+  }
+
+  /**
+   * Returns the documents for the specified remote keys if they still match the query, sorted by
+   * the query's comparator.
+   */
+  private ImmutableSortedSet<Document> getSortedPreviousResults(
+      Query query, ImmutableSortedSet<DocumentKey> remoteKeys) {
+    // Fetch the documents that matched the query at the last snapshot.
+    ImmutableSortedMap<DocumentKey, MaybeDocument> previousResults =
+        localDocumentsView.getDocuments(remoteKeys);
+
+    // Sort the documents and re-apply the query filter since previously matching documents do not
+    // necessarily still match the query.
+    ImmutableSortedSet<Document> results =
+        new ImmutableSortedSet<>(Collections.emptyList(), query.comparator());
+    for (Map.Entry<DocumentKey, MaybeDocument> entry : previousResults) {
+      MaybeDocument maybeDoc = entry.getValue();
+      if (maybeDoc instanceof Document && query.matches((Document) maybeDoc)) {
+        Document doc = (Document) maybeDoc;
+        results = results.insert(doc);
+      }
+    }
+    return results;
+  }
+
+  /**
+   * Determines if a limit query needs to be refilled from cache, making it ineligible for
+   * index-free execution.
+   *
+   * @param sortedPreviousResults The documents that matched the query when it was last
+   *     synchronized, sorted by the query's comparator.
+   * @param remoteKeys The document keys that matched the query at the last snapshot.
+   * @param limboFreeSnapshotVersion The version of the snapshot when the query was last
+   *     synchronized.
+   */
+  private boolean needsRefill(
+      ImmutableSortedSet<Document> sortedPreviousResults,
+      ImmutableSortedSet<DocumentKey> remoteKeys,
+      SnapshotVersion limboFreeSnapshotVersion) {
+    // The query needs to be refilled if a previously matching document no longer matches.
+    if (remoteKeys.size() != sortedPreviousResults.size()) {
+      return true;
+    }
+
+    // We don't need to find a better match from cache if no documents matched the query.
+    if (sortedPreviousResults.isEmpty()) {
+      return false;
+    }
+
+    // Limit queries are not eligible for index-free query execution if there is a potential that an
+    // older document from cache now sorts before a document that was previously part of the limit.
+    // This, however, can only happen if the last document of the limit sorts lower than it did when
+    // the query was last synchronized. If a document that is not the limit boundary sorts
+    // differently, the boundary of the limit itself did not change and documents from cache will
+    // continue to be "rejected" by this boundary. Therefore, we can ignore any modifications that
+    // don't affect the last document.
+    Document lastDocumentInLimit = sortedPreviousResults.getMaxEntry();
+    return lastDocumentInLimit.hasPendingWrites()
+        || lastDocumentInLimit.getVersion().compareTo(limboFreeSnapshotVersion) > 0;
+  }
+
+  @Override
+  public void handleDocumentChange(MaybeDocument oldDocument, MaybeDocument newDocument) {
+    // No indexes to update.
+  }
+
+  private ImmutableSortedMap<DocumentKey, Document> executeFullCollectionScan(Query query) {
+    return localDocumentsView.getDocumentsMatchingQuery(query, SnapshotVersion.NONE);
+  }
+}