Setup Desktop test workflow that detects C++ SDK breakage caused by iOS SDK changes

.github/workflows/integration_tests.yml

@@ -3,7 +3,8 @@ name: Integration tests
 on:
   schedule:
     - cron: "0 9 * * *"  # 9am UTC = 1am PST / 2am PDT. for all testapps except firestore
-    - cron: "0 10 * * *"  # 10am UTC = 2am PST / 3am PDT. for firestore test only
+    - cron: "0 10 * * *"  # 10am UTC = 2am PST / 3am PDT. for firestore test against release iOS/Android SDK
+    - cron: "0 11 * * *"  # 11am UTC = 3am PST / 4am PDT. for firestore desktop test aginst tip-of-tree iOS repo
 
   pull_request:
     types: [ labeled, closed ]
@@ -183,17 +184,25 @@ jobs:
         # To feed input into the job matrix, we first need to convert to a JSON
         # list. Then we can use fromJson to define the field in the matrix for the tests job.
         if [[ "${{ github.event.schedule }}" == "0 9 * * *" ]]; then
-          # at 1am PST / 2am PDT. Running integration tests and generate test report for all testapps except firestore
+          # at 1am PST/2am PDT. Running integration tests and generate test report for all testapps except firestore
           apis="admob,analytics,auth,database,dynamic_links,functions,gma,installations,messaging,remote_config,storage"
-        elif [[ "${{ github.event.schedule }}" == "0 10 * * *" ]]; then
-          # at 2am PST / 3am PDT. Running integration tests for firestore and generate test report
+        elif [[ "${{ github.event.schedule }}" == "0 10 * * *" || "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
+          # at 2am PST/3am PDT and 3am PST/4am PDT. Running integration tests for firestore and generate test report.
           apis="firestore"
         else
           apis=$( python scripts/gha/print_matrix_configuration.py -c -w integration_tests -k apis -o "${{github.event.inputs.apis}}" ${TEST_MATRIX_PARAM} )
         fi
+        if [[ "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
+          # at 3am PST/4am PDT. Running  firestore desktop integration test aginst tip-of-tree ios repo
+          matrix_platform="Desktop"
+          matrix_os=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "ubuntu-20.04,macos-12")
+        else
+          matrix_platform=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k platform -o "${{github.event.inputs.platforms}}" --apis ${apis} )
+          matrix_os=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "${{github.event.inputs.operating_systems}}")
+        fi
         echo "apis=${apis}" >> $GITHUB_OUTPUT
-        echo "matrix_platform=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k platform -o "${{github.event.inputs.platforms}}" --apis ${apis} )" >> $GITHUB_OUTPUT
-        echo "matrix_os=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "${{github.event.inputs.operating_systems}}")" >> $GITHUB_OUTPUT
+        echo "matrix_platform=${matrix_platform}" >> $GITHUB_OUTPUT
+        echo "matrix_os=${matrix_os}" >> $GITHUB_OUTPUT
         echo "matrix_arch_macos=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_macos)" >> $GITHUB_OUTPUT
         echo "matrix_arch_windows_linux=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_windows_linux)" >> $GITHUB_OUTPUT
         # Combine architecture_macos and architecture_windows_linux to get a list of all architectures for the build matrix.
@@ -318,6 +327,10 @@ jobs:
               additional_flags+=(--cmake_flag=-DFIREBASE_USE_BORINGSSL=ON)
             fi
           fi
+          if [[ "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
+            # at 3am PST/4am PDT. Running firestore desktop integration test aginst tip-of-tree ios repo
+            additional_flags+=(--cmake_flag=-DFIRESTORE_DEP_SOURCE=TIP)
+          fi
           python scripts/gha/build_testapps.py --p Desktop \
             --t ${{ needs.check_and_prepare.outputs.apis }} \
             --output_directory "${{ github.workspace }}" \
@@ -327,6 +340,13 @@ jobs:
             --gha_build \
             --arch ${{ matrix.arch }} \
             ${additional_flags[*]}
+      - name: Upload Desktop Cmake
+        uses: actions/upload-artifact@v3
+        if: ${{ !cancelled() }}
+        with:
+          name: cmake-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}
+          path: D:/a/firebase-cpp-sdk/firebase-cpp-sdk/ta/firestore/iti/CMakeFiles/
+          retention-days: 1
       - name: Prepare results summary artifact
         if: ${{ !cancelled() }}
         shell: bash
@@ -1361,7 +1381,12 @@ jobs:
           if [[ "${{ github.event.inputs.test_pull_request }}" == "nightly-packaging" ]]; then
             additional_flags=(--build_against sdk)
           else
-            additional_flags=(--build_against repo)
+            if [[ "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
+              # at 3am PST/4am PDT. Running firestore desktop integration test aginst tip-of-tree ios repo
+              additional_flags=(--build_against tip)
+            else
+              additional_flags=(--build_against repo)
+            fi
           fi
           if [[ "${{ needs.check_and_prepare.outputs.apis }}" == "firestore" ]]; then
             additional_flags+=(--build_apis firestore)

scripts/gha/build_desktop.py

@@ -230,7 +230,7 @@ def cmake_configure(build_dir, arch, msvc_runtime_library='static', linux_abi='l
   if verbose:
     cmd.append('-DCMAKE_VERBOSE_MAKEFILE=1')
 
-  utils.run_command(cmd)
+  utils.run_command(cmd, check=True)
 
 def main():
   args = parse_cmdline_args()
@@ -269,7 +269,7 @@ def main():
     # Example:  cmake --build build -j 8 --target firebase_app firebase_auth
     cmd.append('--target')
     cmd.extend(args.target)
-  utils.run_command(cmd)
+  utils.run_command(cmd, check=True)
 
 
 def parse_cmdline_args():

scripts/gha/it_workflow.py

@@ -66,17 +66,20 @@
 _COMMENT_TITLE_FLAKY = "### Integration test with FLAKINESS (succeeded after retry)\n"
 _COMMENT_TITLE_FAIL = "### ❌  Integration test FAILED\n"
 _COMMENT_TITLE_SUCCEED = "### ✅  Integration test succeeded!\n"
-_COMMENT_TITLE_FLAKY_SDK = "\n***\n### [build against SDK] Integration test with FLAKINESS (succeeded after retry)\n"
-_COMMENT_TITLE_FAIL_SDK = "\n***\n### ❌  [build against SDK] Integration test FAILED\n"
-_COMMENT_TITLE_SUCCEED_SDK = "\n***\n### ✅  [build against SDK] Integration test succeeded!\n"
 _COMMENT_TITLE_FLAKY_REPO = "### [build against repo] Integration test with FLAKINESS (succeeded after retry)\n"
 _COMMENT_TITLE_FAIL_REPO = "### ❌  [build against repo] Integration test FAILED\n"
 _COMMENT_TITLE_SUCCEED_REPO = "### ✅  [build against repo] Integration test succeeded!\n"
+_COMMENT_TITLE_FLAKY_SDK = "\n***\n### [build against SDK] Integration test with FLAKINESS (succeeded after retry)\n"
+_COMMENT_TITLE_FAIL_SDK = "\n***\n### ❌  [build against SDK] Integration test FAILED\n"
+_COMMENT_TITLE_SUCCEED_SDK = "\n***\n### ✅  [build against SDK] Integration test succeeded!\n"
+_COMMENT_TITLE_FLAKY_TIP = "\n***\n### [build against tip] Integration test with FLAKINESS (succeeded after retry)\n"
+_COMMENT_TITLE_FAIL_TIP = "\n***\n### ❌  [build against tip] Integration test FAILED\n"
+_COMMENT_TITLE_SUCCEED_TIP = "\n***\n### ✅  [build against tip] Integration test succeeded!\n"
 
 _COMMENT_FLAKY_TRACKER = "\n\nAdd flaky tests to **[go/fpl-cpp-flake-tracker](http://go/fpl-cpp-flake-tracker)**\n"
 
 _COMMENT_IDENTIFIER = "integration-test-status-comment"
-_COMMENT_HIDDEN_DIVIDER = f'\n<hidden value="{_COMMENT_IDENTIFIER}"></hidden>\n'
+_COMMENT_HIDDEN_DIVIDER = f'\r\n<hidden value="{_COMMENT_IDENTIFIER}"></hidden>\r\n'
 
 _LOG_ARTIFACT_NAME = "log-artifact"
 _LOG_OUTPUT_DIR = "test_results"
@@ -89,6 +92,7 @@
 
 _BUILD_AGAINST_SDK = "sdk"
 _BUILD_AGAINST_REPO = "repo"
+_BUILD_AGAINST_TIP = "tip"
 
 _BUILD_API_ALL = "all"
 _BUILD_API_FIRESTORE = "firestore"
@@ -222,27 +226,44 @@ def test_report(token, actor, commit, run_id, build_against, build_apis):
 
   issue_number = _get_issue_number(token, report_title, _REPORT_LABEL)
   previous_comment = github.get_issue_body(token, issue_number)
-  [_, previous_comment_repo, previous_comment_sdk] = previous_comment.split(_COMMENT_HIDDEN_DIVIDER)
+  [_, previous_comment_repo, previous_comment_sdk, previous_comment_tip] = previous_comment.split(_COMMENT_HIDDEN_DIVIDER)
   success_or_only_flakiness, log_summary = _get_summary_table(token, run_id)
   if success_or_only_flakiness and not log_summary:
     # succeeded (without flakiness)
-    title = _COMMENT_TITLE_SUCCEED_REPO if build_against==_BUILD_AGAINST_REPO else _COMMENT_TITLE_SUCCEED_SDK
+    if build_against==_BUILD_AGAINST_REPO:
+      title = _COMMENT_TITLE_SUCCEED_REPO  
+    elif build_against==_BUILD_AGAINST_SDK:
+      title = _COMMENT_TITLE_SUCCEED_SDK
+    else:
+      title = _COMMENT_TITLE_SUCCEED_TIP
     comment = title + _get_description(actor, commit, run_id)
   else:
     if success_or_only_flakiness:
       # all failures/errors are due to flakiness (succeeded after retry)
-      title = _COMMENT_TITLE_FLAKY_REPO if build_against==_BUILD_AGAINST_REPO else _COMMENT_TITLE_FLAKY_SDK
+      if build_against==_BUILD_AGAINST_REPO:
+        title = _COMMENT_TITLE_FLAKY_REPO  
+      elif build_against==_BUILD_AGAINST_SDK:
+        title = _COMMENT_TITLE_FLAKY_SDK
+      else:
+        title = _COMMENT_TITLE_FLAKY_TIP
     else:
       # failures/errors still exist after retry
-      title = _COMMENT_TITLE_FAIL_REPO if build_against==_BUILD_AGAINST_REPO else _COMMENT_TITLE_FAIL_SDK
+      if build_against==_BUILD_AGAINST_REPO:
+        title = _COMMENT_TITLE_FAIL_REPO  
+      elif build_against==_BUILD_AGAINST_SDK:
+        title = _COMMENT_TITLE_FAIL_SDK
+      else:
+        title = _COMMENT_TITLE_FAIL_TIP
     comment = title + _get_description(actor, commit, run_id) + log_summary + _COMMENT_FLAKY_TRACKER
 
   if build_against==_BUILD_AGAINST_REPO:
-    comment = prefix + _COMMENT_HIDDEN_DIVIDER + comment + _COMMENT_HIDDEN_DIVIDER + previous_comment_sdk
+    comment = prefix + _COMMENT_HIDDEN_DIVIDER + comment + _COMMENT_HIDDEN_DIVIDER + previous_comment_sdk + _COMMENT_HIDDEN_DIVIDER + previous_comment_tip
   elif build_against==_BUILD_AGAINST_SDK:
-    comment = prefix + _COMMENT_HIDDEN_DIVIDER + previous_comment_repo + _COMMENT_HIDDEN_DIVIDER + comment
+    comment = prefix + _COMMENT_HIDDEN_DIVIDER + previous_comment_repo + _COMMENT_HIDDEN_DIVIDER + comment + _COMMENT_HIDDEN_DIVIDER + previous_comment_tip
+  else:
+    comment = prefix + _COMMENT_HIDDEN_DIVIDER + previous_comment_repo + _COMMENT_HIDDEN_DIVIDER + previous_comment_sdk + _COMMENT_HIDDEN_DIVIDER + comment
 
-  if (_COMMENT_TITLE_SUCCEED_REPO in comment) and (_COMMENT_TITLE_SUCCEED_SDK in comment):
+  if (_COMMENT_TITLE_SUCCEED_REPO in comment) and (_COMMENT_TITLE_SUCCEED_SDK in comment) and (build_apis != _BUILD_API_FIRESTORE or _COMMENT_TITLE_SUCCEED_TIP in comment):
     github.close_issue(token, issue_number)
   else:
     github.open_issue(token, issue_number)