Run Firestore CI from Android Emulator instead (#6094)

Run Firestore Integration tests from Android Emulator, this brings down
the tests runtime to around 10 minutes from 40.

There are several changes in this PR:
1. Take firestore out of ci_tests.yml. 
2. Create firestore_ci_tests.yml so that other SDKs do not have to setup
emulators.
3. Revert #6057.
This PR is causing failures like:
https://github.com/firebase/firebase-android-sdk/actions/runs/9910054539/job/27379655403?pr=6094
after we switch to emulator, may be related to its more concurrent
nature.
4. Commented out a test that fails on CI.

Author: wu-hui

.github/workflows/ci_tests.yml

@@ -112,6 +112,8 @@ jobs:
       fail-fast: false
       matrix:
         module: ${{ fromJSON(needs.determine_changed.outputs.modules) }}
+        exclude:
+          - module: :firebase-firestore
 
     steps:
       - uses: actions/[email protected]
@@ -144,119 +146,6 @@ jobs:
         run: |
           ./gradlew ${{matrix.module}}:deviceCheck withErrorProne -PtargetBackend="prod"
 
-  firestore_custom_integ_tests:
-    name: "Firestore Custom Instrumentation Tests Against Named DB"
-    runs-on: ubuntu-22.04
-    needs:
-      - determine_changed
-    # only run on post submit or PRs not originating from forks.
-    if: ((github.repository == 'Firebase/firebase-android-sdk' && github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) && contains(fromJSON(needs.determine_changed.outputs.modules), ':firebase-firestore')
-    strategy:
-      fail-fast: false
-
-    steps:
-      - uses: actions/[email protected]
-        with:
-          fetch-depth: 2
-          submodules: true
-
-      - name: Set up JDK 17
-        uses: actions/[email protected]
-        with:
-          java-version: 17
-          distribution: temurin
-          cache: gradle
-
-      - name: Add google-services.json
-        env:
-          INTEG_TESTS_GOOGLE_SERVICES: ${{ secrets.INTEG_TESTS_GOOGLE_SERVICES }}
-        run: |
-          echo $INTEG_TESTS_GOOGLE_SERVICES | base64 -d > google-services.json
-      - uses: google-github-actions/auth@v2
-        with:
-          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
-      - uses: google-github-actions/setup-gcloud@v2
-
-      # create composite indexes with Terraform
-      - name: Setup Terraform
-        uses: hashicorp/setup-terraform@v2
-      - name: Terraform Init
-        run: |
-          cd firebase-firestore
-          terraform init
-        continue-on-error: true
-      - name: Terraform Apply
-        if: github.event_name == 'pull_request'
-        run: |
-          cd firebase-firestore
-
-          # Define a temporary file, redirect both stdout and stderr to the file
-          output_file=$(mktemp)
-          if ! terraform apply -var-file=../google-services.json -auto-approve > "$output_file" 2>&1 ; then
-            cat "$output_file"
-            if cat "$output_file" | grep -q "index already exists"; then
-              echo "==================================================================================="
-              echo -e "\e[93m\e[1mTerraform apply failed due to index already exists; We can safely ignore this error.\e[0m"
-              echo "==================================================================================="
-            fi
-            exit 1
-          fi
-          rm -f "$output_file"
-        continue-on-error: true
-
-      - name: Firestore Named DB Integ Tests
-        env:
-          FIREBASE_CI: 1
-          FTL_RESULTS_BUCKET: android-ci
-          FTL_RESULTS_DIR: ${{ github.event_name == 'pull_request' && format('pr-logs/pull/{0}/{1}/{2}/{3}_{4}/artifacts/', github.repository, github.event.pull_request.number, github.job, github.run_id, github.run_attempt) || format('logs/{0}/{1}_{2}/artifacts/', github.workflow, github.run_id, github.run_attempt)}}
-          FIREBASE_APP_CHECK_DEBUG_SECRET: ${{ secrets.FIREBASE_APP_CHECK_DEBUG_SECRET }}
-        run: |
-          ./gradlew firebase-firestore:deviceCheck withErrorProne -PtargetBackend="prod" -PtargetDatabaseId="test-db"
-
-
-  firestore_nightly_integ_tests:
-    name: "Firestore Instrumentation Tests Against Nightly Environment"
-    runs-on: ubuntu-22.04
-    needs:
-      - determine_changed
-    # only run on post submit or PRs not originating from forks.
-    if: ((github.repository == 'Firebase/firebase-android-sdk' && github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) && contains(fromJSON(needs.determine_changed.outputs.modules), ':firebase-firestore')
-    strategy:
-      fail-fast: false
-
-    steps:
-      - uses: actions/[email protected]
-        with:
-          fetch-depth: 2
-          submodules: true
-
-      - name: Set up JDK 17
-        uses: actions/[email protected]
-        with:
-          java-version: 17
-          distribution: temurin
-          cache: gradle
-
-      - name: Add google-services.json
-        env:
-          INTEG_TESTS_GOOGLE_SERVICES: ${{ secrets.NIGHTLY_INTEG_TESTS_GOOGLE_SERVICES }}
-        run: |
-          echo $INTEG_TESTS_GOOGLE_SERVICES > google-services.json
-      - uses: google-github-actions/auth@v2
-        with:
-          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
-      - uses: google-github-actions/setup-gcloud@v2
-
-      - name: Firestore Nightly Integ Tests
-        env:
-          FIREBASE_CI: 1
-          FTL_RESULTS_BUCKET: android-ci
-          FTL_RESULTS_DIR: ${{ github.event_name == 'pull_request' && format('pr-logs/pull/{0}/{1}/{2}/{3}_{4}/artifacts/', github.repository, github.event.pull_request.number, github.job, github.run_id, github.run_attempt) || format('logs/{0}/{1}_{2}/artifacts/', github.workflow, github.run_id, github.run_attempt)}}
-          FIREBASE_APP_CHECK_DEBUG_SECRET: ${{ secrets.FIREBASE_APP_CHECK_DEBUG_SECRET }}
-        run: |
-          ./gradlew firebase-firestore:deviceCheck withErrorProne -PtargetBackend="nightly"
-
-
   publish-test-results:
     name: "Publish Tests Results"
     needs:

.github/workflows/firestore_ci_tests.yml

@@ -0,0 +1,261 @@
+name: Firestore CI Tests
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  determine_changed:
+    name: "Determine changed modules"
+    runs-on: ubuntu-latest
+    if: (github.repository == 'Firebase/firebase-android-sdk' && github.event_name == 'push') || github.event_name == 'pull_request'
+    outputs:
+      modules: ${{ steps.changed-modules.outputs.modules }}
+    steps:
+      - uses: actions/[email protected]
+        with:
+          fetch-depth: 2
+          submodules: true
+
+      - name: Set up JDK 17
+        uses: actions/[email protected]
+        with:
+          java-version: 17
+          distribution: temurin
+          cache: gradle
+
+      - id: changed-modules
+        run: |
+          git diff --name-only HEAD~1 | xargs printf -- '--changed-git-paths %s\n' | xargs ./gradlew writeChangedProjects --output-file-path=modules.json
+          echo modules=$(cat modules.json) >> $GITHUB_OUTPUT
+
+  integ_tests:
+    name: "System Tests"
+    # only run on post submit or PRs not originating from forks.
+    if: ((github.repository == 'Firebase/firebase-android-sdk' && github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) && contains(fromJSON(needs.determine_changed.outputs.modules), ':firebase-firestore')
+    runs-on: ubuntu-latest
+    needs:
+      - determine_changed
+    strategy:
+      fail-fast: false
+
+    steps:
+      - uses: actions/[email protected]
+        with:
+          fetch-depth: 2
+          submodules: true
+
+      - name: Enable KVM
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm    
+
+      - name: Set up JDK 17
+        uses: actions/[email protected]
+        with:
+          java-version: 17
+          distribution: temurin
+          cache: gradle
+
+      - name: Add google-services.json
+        env:
+          INTEG_TESTS_GOOGLE_SERVICES: ${{ secrets.INTEG_TESTS_GOOGLE_SERVICES }}
+        run: |
+          echo $INTEG_TESTS_GOOGLE_SERVICES | base64 -d > google-services.json
+      - uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+      - uses: google-github-actions/setup-gcloud@v2
+      - name: firebase-firestore Integ Tests
+        uses: reactivecircus/android-emulator-runner@v2
+        env:
+          FIREBASE_CI: 1
+          FTL_RESULTS_BUCKET: android-ci
+          FTL_RESULTS_DIR: ${{ github.event_name == 'pull_request' && format('pr-logs/pull/{0}/{1}/{2}/{3}_{4}/artifacts/', github.repository, github.event.pull_request.number, github.job, github.run_id, github.run_attempt) || format('logs/{0}/{1}_{2}/artifacts/', github.workflow, github.run_id, github.run_attempt)}}
+          FIREBASE_APP_CHECK_DEBUG_SECRET: ${{ secrets.FIREBASE_APP_CHECK_DEBUG_SECRET }}
+        with:
+          api-level: 31
+          arch: x86_64
+          ram-size: 4096M
+          heap-size: 4096M
+          script: |
+            adb logcat -v time > logcat.txt &
+            ./gradlew firebase-firestore:connectedCheck withErrorProne -PtargetBackend="prod"
+      - name: Upload logs
+        if: failure()
+        uses: actions/[email protected]
+        with:
+          name: logcat.txt
+          path: logcat.txt
+          retention-days: 7
+          if-no-files-found: ignore
+
+
+  named_integ_tests:
+    name: "System Tests With Named DB"
+    runs-on: ubuntu-latest
+    needs:
+      - determine_changed
+    # only run on post submit or PRs not originating from forks.
+    if: ((github.repository == 'Firebase/firebase-android-sdk' && github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) && contains(fromJSON(needs.determine_changed.outputs.modules), ':firebase-firestore')
+    strategy:
+      fail-fast: false
+
+    steps:
+      - uses: actions/[email protected]
+        with:
+          fetch-depth: 2
+          submodules: true
+
+      - name: Enable KVM
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm    
+
+      - name: Set up JDK 17
+        uses: actions/[email protected]
+        with:
+          java-version: 17
+          distribution: temurin
+          cache: gradle
+
+      - name: Add google-services.json
+        env:
+          INTEG_TESTS_GOOGLE_SERVICES: ${{ secrets.INTEG_TESTS_GOOGLE_SERVICES }}
+        run: |
+          echo $INTEG_TESTS_GOOGLE_SERVICES | base64 -d > google-services.json
+      - uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+      - uses: google-github-actions/setup-gcloud@v2
+
+      # create composite indexes with Terraform
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v2
+      - name: Terraform Init
+        run: |
+          cd firebase-firestore
+          terraform init
+        continue-on-error: true
+      - name: Terraform Apply
+        if: github.event_name == 'pull_request'
+        run: |
+          cd firebase-firestore
+
+          # Define a temporary file, redirect both stdout and stderr to the file
+          output_file=$(mktemp)
+          if ! terraform apply -var-file=../google-services.json -auto-approve > "$output_file" 2>&1 ; then
+            cat "$output_file"
+            if cat "$output_file" | grep -q "index already exists"; then
+              echo "==================================================================================="
+              echo -e "\e[93m\e[1mTerraform apply failed due to index already exists; We can safely ignore this error.\e[0m"
+              echo "==================================================================================="
+            fi
+            exit 1
+          fi
+          rm -f "$output_file"
+        continue-on-error: true
+
+      - name: Firestore Named DB Integ Tests
+        timeout-minutes: 20
+        uses: reactivecircus/android-emulator-runner@v2
+        env:
+          FIREBASE_CI: 1
+          FTL_RESULTS_BUCKET: android-ci
+          FTL_RESULTS_DIR: ${{ github.event_name == 'pull_request' && format('pr-logs/pull/{0}/{1}/{2}/{3}_{4}/artifacts/', github.repository, github.event.pull_request.number, github.job, github.run_id, github.run_attempt) || format('logs/{0}/{1}_{2}/artifacts/', github.workflow, github.run_id, github.run_attempt)}}
+          FIREBASE_APP_CHECK_DEBUG_SECRET: ${{ secrets.FIREBASE_APP_CHECK_DEBUG_SECRET }}
+        with:
+          api-level: 31
+          arch: x86_64
+          ram-size: 4096M
+          heap-size: 4096M
+          script: |
+            adb logcat -v time > logcat.txt &
+            ./gradlew firebase-firestore:connectedCheck withErrorProne -PtargetBackend="prod"
+      - name: Upload logs
+        if: failure()
+        uses: actions/[email protected]
+        with:
+          name: named-db-logcat.txt
+          path: logcat.txt
+          retention-days: 7
+          if-no-files-found: ignore
+
+  firestore_nightly_integ_tests:
+    name: "System Tests Against Nightly"
+    runs-on: ubuntu-latest
+    needs:
+      - determine_changed
+    # only run on post submit or PRs not originating from forks.
+    if: ((github.repository == 'Firebase/firebase-android-sdk' && github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) && contains(fromJSON(needs.determine_changed.outputs.modules), ':firebase-firestore')
+    strategy:
+      fail-fast: false
+
+    steps:
+      - uses: actions/[email protected]
+        with:
+          fetch-depth: 2
+          submodules: true
+
+      - name: Enable KVM
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm    
+
+      - name: Set up JDK 17
+        uses: actions/[email protected]
+        with:
+          java-version: 17
+          distribution: temurin
+          cache: gradle
+
+      - name: Add google-services.json
+        env:
+          INTEG_TESTS_GOOGLE_SERVICES: ${{ secrets.NIGHTLY_INTEG_TESTS_GOOGLE_SERVICES }}
+        run: |
+          echo $INTEG_TESTS_GOOGLE_SERVICES > google-services.json
+      - uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+      - uses: google-github-actions/setup-gcloud@v2
+
+      - name: Firestore Nightly Integ Tests
+        uses: reactivecircus/android-emulator-runner@v2
+        env:
+          FIREBASE_CI: 1
+          FTL_RESULTS_BUCKET: android-ci
+          FTL_RESULTS_DIR: ${{ github.event_name == 'pull_request' && format('pr-logs/pull/{0}/{1}/{2}/{3}_{4}/artifacts/', github.repository, github.event.pull_request.number, github.job, github.run_id, github.run_attempt) || format('logs/{0}/{1}_{2}/artifacts/', github.workflow, github.run_id, github.run_attempt)}}
+          FIREBASE_APP_CHECK_DEBUG_SECRET: ${{ secrets.FIREBASE_APP_CHECK_DEBUG_SECRET }}
+        with:
+          api-level: 31
+          arch: x86_64
+          ram-size: 4096M
+          heap-size: 4096M
+          script: |
+            adb logcat -v time > logcat.txt &
+            ./gradlew firebase-firestore:connectedCheck withErrorProne -PtargetBackend="nightly"
+      - name: Upload logs
+        if: failure()
+        uses: actions/[email protected]
+        with:
+          name: nightly-logcat.txt
+          path: logcat.txt
+          retention-days: 7
+          if-no-files-found: ignore
+
+  check-required-tests:
+    runs-on: ubuntu-latest
+    if: always()
+    name: Check all required Firestore tests results
+    needs: [integ_tests, named_integ_tests]
+    steps:
+      - name: Check test matrix
+        if: needs.integ_tests.result == 'failure' || needs.named_integ_tests.result == 'failure'
+        run: exit 1