Testing composite index queries against production

.github/workflows/ci_tests.yml

@@ -121,6 +121,24 @@ jobs:
         with:
           credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
       - uses: google-github-actions/setup-gcloud@v0
+      # create composite indexes with Terraform
+      - name: Setup Terraform 
+        if: contains(matrix.module, ':firebase-firestore')
+        uses: hashicorp/setup-terraform@v2
+      - name: Terraform Init
+        if: contains(matrix.module, ':firebase-firestore')
+        run: |
+          pwd
+          cd firebase-firestore
+          terraform init
+        continue-on-error: true
+      - name: Terraform Apply
+        if: github.event_name == 'pull_request' && contains(matrix.module, ':firebase-firestore')
+        run: |
+          pwd
+          cd firebase-firestore
+          terraform apply -var-file=../google-services.json -auto-approve
+        continue-on-error: true
       - name: ${{ matrix.module }} Integ Tests
         env:
           FIREBASE_CI: 1

.gitignore

@@ -13,3 +13,9 @@ smoke-test-logs/
 smoke-tests/build-debug-headGit-smoke-test
 smoke-tests/firehorn.log
 macrobenchmark-output.json
+
+# generated Terraform docs
+.terraform/*
+.terraform.lock.hcl
+*.tfstate
+*.tfstate.*

firebase-firestore/firestore_index_config.tf

@@ -0,0 +1,104 @@
+locals {
+  indexes = {
+    index1 = [
+      {
+        field_path = "testId"
+        order      = "ASCENDING"
+      },
+      {
+        field_path = "a"
+        order      = "ASCENDING"
+      },
+    ]
+    index2 = [
+      {
+        field_path = "testId"
+        order      = "ASCENDING"
+      },
+      {
+        field_path = "b"
+        order      = "ASCENDING"
+      },
+    ]
+    index3 = [
+      {
+        field_path = "testId"
+        order      = "ASCENDING"
+      },
+      {
+        field_path = "b"
+        order      = "DESCENDING"
+      },
+    ]
+    index4 = [
+      {
+        field_path = "a"
+        order      = "ASCENDING"
+      },
+      {
+        field_path = "testId"
+        order      = "ASCENDING"
+      },
+      {
+        field_path = "b"
+        order      = "ASCENDING"
+      },
+    ]
+    index5 = [
+      {
+        field_path = "a"
+        order      = "ASCENDING"
+      },
+      {
+        field_path = "testId"
+        order      = "ASCENDING"
+      },
+      {
+        field_path = "b"
+        order      = "DESCENDING"
+      },
+    ]
+    index6 = [
+      {
+        field_path = "a"
+        order      = "ASCENDING"
+      },
+      {
+        field_path = "testId"
+        order      = "ASCENDING"
+      },
+      {
+        field_path = "a"
+        order      = "DESCENDING"
+      },
+    ]
+    index7 = [
+      {
+        field_path = "b"
+        order      = "ASCENDING"
+      },
+      {
+        field_path = "testId"
+        order      = "ASCENDING"
+      },
+      {
+        field_path = "a"
+        order      = "ASCENDING"
+      },
+    ]
+    index8 = [
+      {
+        field_path = "b"
+        order      = "ASCENDING"
+      },
+      {
+        field_path = "testId"
+        order      = "ASCENDING"
+      },
+      {
+        field_path = "a"
+        order      = "DESCENDING"
+      },
+    ]
+  }
+}

firebase-firestore/main.tf

@@ -0,0 +1,41 @@
+variable "project_info" {}
+
+provider "google" {
+  project = var.project_info.project_id
+}
+
+resource "google_firestore_index" "default-db-index" {
+  collection = "composite-index-test-collection"
+
+  for_each = local.indexes
+  dynamic "fields" {
+    for_each = distinct(flatten([for k, v in local.indexes : [
+      for i in each.value : {
+        field_path = i.field_path
+        order      = i.order
+    }]]))
+    content {
+      field_path = lookup(fields.value, "field_path", null)
+      order      = lookup(fields.value, "order", null)
+    }
+  }
+
+}
+
+resource "google_firestore_index" "named-db-index" {
+  collection = "composite-index-test-collection"
+  database   = "test-db"
+
+  for_each = local.indexes
+  dynamic "fields" {
+    for_each = distinct(flatten([for k, v in local.indexes : [
+      for i in each.value : {
+        field_path = i.field_path
+        order      = i.order
+    }]]))
+    content {
+      field_path = lookup(fields.value, "field_path", null)
+      order      = lookup(fields.value, "order", null)
+    }
+  }
+}

firebase-firestore/src/androidTest/java/com/google/firebase/firestore/CompositeIndexQueryTest.java

@@ -0,0 +1,83 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.firebase.firestore;
+
+import static com.google.firebase.firestore.Filter.equalTo;
+import static com.google.firebase.firestore.Filter.greaterThan;
+import static com.google.firebase.firestore.Filter.or;
+import static com.google.firebase.firestore.testutil.TestUtil.map;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.firebase.firestore.testutil.CompositeIndexTestHelper;
+import com.google.firebase.firestore.testutil.IntegrationTestUtil;
+import java.util.Map;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/*
+ * Guidance for Creating Tests:
+ * ----------------------------
+ * When creating tests that require composite indexes, it is recommended to utilize the
+ * "CompositeIndexTestHelper" class. This utility class provides methods for creating
+ * and setting test documents and running queries with ease, ensuring proper data
+ * isolation and query construction.
+ *
+ * Please remember to update the main index configuration file (firestore_index_config.tf)
+ * with any new composite indexes needed for the tests. This ensures synchronization with
+ * other testing environments, including CI. You can generate the required index link by
+ * clicking on the Firebase console link in the error message while running tests locally.
+ */
+@RunWith(AndroidJUnit4.class)
+public class CompositeIndexQueryTest {
+
+  @After
+  public void tearDown() {
+    IntegrationTestUtil.tearDown();
+  }
+
+  @Test
+  public void testOrQueriesWithCompositeIndexes() {
+    CompositeIndexTestHelper testHelper = new CompositeIndexTestHelper();
+    Map<String, Map<String, Object>> testDocs =
+        map(
+            "doc1", map("a", 1, "b", 0),
+            "doc2", map("a", 2, "b", 1),
+            "doc3", map("a", 3, "b", 2),
+            "doc4", map("a", 1, "b", 3),
+            "doc5", map("a", 1, "b", 1));
+    CollectionReference collection = testHelper.withTestDocs(testDocs);
+
+    Query query = collection.where(or(greaterThan("a", 2), equalTo("b", 1)));
+    // with one inequality: a>2 || b==1.
+    testHelper.assertOnlineAndOfflineResultsMatch(testHelper.query(query), "doc5", "doc2", "doc3");
+
+    // Test with limits (implicit order by ASC): (a==1) || (b > 0) LIMIT 2
+    query = collection.where(or(equalTo("a", 1), greaterThan("b", 0))).limit(2);
+    testHelper.assertOnlineAndOfflineResultsMatch(testHelper.query(query), "doc1", "doc2");
+
+    // Test with limits (explicit order by): (a==1) || (b > 0) LIMIT_TO_LAST 2
+    // Note: The public query API does not allow implicit ordering when limitToLast is used.
+    query = collection.where(or(equalTo("a", 1), greaterThan("b", 0))).limitToLast(2).orderBy("b");
+    testHelper.assertOnlineAndOfflineResultsMatch(testHelper.query(query), "doc3", "doc4");
+
+    // Test with limits (explicit order by ASC): (a==2) || (b == 1) ORDER BY a LIMIT 1
+    query = collection.where(or(equalTo("a", 2), equalTo("b", 1))).limit(1).orderBy("a");
+    testHelper.assertOnlineAndOfflineResultsMatch(testHelper.query(query), "doc5");
+
+    // Test with limits (explicit order by DESC): (a==2) || (b == 1) ORDER BY a LIMIT_TO_LAST 1
+    query = collection.where(or(equalTo("a", 2), equalTo("b", 1))).limitToLast(1).orderBy("a");
+    testHelper.assertOnlineAndOfflineResultsMatch(testHelper.query(query), "doc2");
+  }
+}

firebase-firestore/src/androidTest/java/com/google/firebase/firestore/QueryTest.java

@@ -29,6 +29,7 @@
 import static com.google.firebase.firestore.Filter.notInArray;
 import static com.google.firebase.firestore.Filter.or;
 import static com.google.firebase.firestore.remote.TestingHooksUtil.captureExistenceFilterMismatches;
+import static com.google.firebase.firestore.testutil.IntegrationTestUtil.checkOnlineAndOfflineResultsMatch;
 import static com.google.firebase.firestore.testutil.IntegrationTestUtil.isRunningAgainstEmulator;
 import static com.google.firebase.firestore.testutil.IntegrationTestUtil.nullList;
 import static com.google.firebase.firestore.testutil.IntegrationTestUtil.querySnapshotToIds;
@@ -78,25 +79,6 @@ public void tearDown() {
     IntegrationTestUtil.tearDown();
   }
 
-  /**
-   * Checks that running the query while online (against the backend/emulator) results in the same
-   * documents as running the query while offline. If `expectedDocs` is provided, it also checks
-   * that both online and offline query result is equal to the expected documents.
-   *
-   * @param query The query to check
-   * @param expectedDocs Ordered list of document keys that are expected to match the query
-   */
-  public void checkOnlineAndOfflineResultsMatch(Query query, String... expectedDocs) {
-    QuerySnapshot docsFromServer = waitFor(query.get(Source.SERVER));
-    QuerySnapshot docsFromCache = waitFor(query.get(Source.CACHE));
-
-    assertEquals(querySnapshotToIds(docsFromServer), querySnapshotToIds(docsFromCache));
-    List<String> expected = asList(expectedDocs);
-    if (!expected.isEmpty()) {
-      assertEquals(expected, querySnapshotToIds(docsFromCache));
-    }
-  }
-
   @Test
   public void testLimitQueries() {
     CollectionReference collection =
@@ -1522,46 +1504,6 @@ public void testOrQueries() {
         collection.where(or(equalTo("a", 2), equalTo("b", 1))).limit(1), "doc2");
   }
 
-  @Test
-  public void testOrQueriesWithCompositeIndexes() {
-    assumeTrue(
-        "Skip this test if running against production because it results in a "
-            + "'missing index' error. The Firestore Emulator, however, does serve these "
-            + " queries.",
-        isRunningAgainstEmulator());
-    Map<String, Map<String, Object>> testDocs =
-        map(
-            "doc1", map("a", 1, "b", 0),
-            "doc2", map("a", 2, "b", 1),
-            "doc3", map("a", 3, "b", 2),
-            "doc4", map("a", 1, "b", 3),
-            "doc5", map("a", 1, "b", 1));
-    CollectionReference collection = testCollectionWithDocs(testDocs);
-
-    // with one inequality: a>2 || b==1.
-    checkOnlineAndOfflineResultsMatch(
-        collection.where(or(greaterThan("a", 2), equalTo("b", 1))), "doc5", "doc2", "doc3");
-
-    // Test with limits (implicit order by ASC): (a==1) || (b > 0) LIMIT 2
-    checkOnlineAndOfflineResultsMatch(
-        collection.where(or(equalTo("a", 1), greaterThan("b", 0))).limit(2), "doc1", "doc2");
-
-    // Test with limits (explicit order by): (a==1) || (b > 0) LIMIT_TO_LAST 2
-    // Note: The public query API does not allow implicit ordering when limitToLast is used.
-    checkOnlineAndOfflineResultsMatch(
-        collection.where(or(equalTo("a", 1), greaterThan("b", 0))).limitToLast(2).orderBy("b"),
-        "doc3",
-        "doc4");
-
-    // Test with limits (explicit order by ASC): (a==2) || (b == 1) ORDER BY a LIMIT 1
-    checkOnlineAndOfflineResultsMatch(
-        collection.where(or(equalTo("a", 2), equalTo("b", 1))).limit(1).orderBy("a"), "doc5");
-
-    // Test with limits (explicit order by DESC): (a==2) || (b == 1) ORDER BY a LIMIT_TO_LAST 1
-    checkOnlineAndOfflineResultsMatch(
-        collection.where(or(equalTo("a", 2), equalTo("b", 1))).limitToLast(1).orderBy("a"), "doc2");
-  }
-
   @Test
   public void testOrQueriesWithIn() {
     Map<String, Map<String, Object>> testDocs =