Merge branch 'releases/m163' into releases/m163.release

Author: daymxn

.github/actions/dataconnect-send-notifications/action.yml

@@ -0,0 +1,71 @@
+name: Data Connect Workflow Notifications
+description: Notify a GitHub Issue with the results of a workflow.
+
+inputs:
+  python-version:
+    required: true
+    default: "3.13"
+  github-issue-for-scheduled-runs:
+    required: true
+  job-results-file:
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - run: pip install -r requirements.txt
+      shell: bash
+      working-directory: firebase-dataconnect/ci
+
+    - id: issue-id
+      name: Determine GitHub Issue For Commenting
+      working-directory: firebase-dataconnect/ci
+      shell: bash
+      run: |
+        args=(
+          python
+          calculate_github_issue_for_commenting.py
+          --issue-output-file=github_issue_number.txt
+          --github-repository='${{ github.repository }}'
+          --github-ref='${{ github.ref }}'
+          --github-event-name='${{ github.event_name }}'
+          --pr-body-github-issue-key=trksmnkncd_notification_issue
+          --github-issue-for-scheduled-run='${{ inputs.github-issue-for-scheduled-runs }}'
+        )
+        echo "${args[*]}"
+        "${args[@]}"
+
+        set -xv
+        issue="$(cat github_issue_number.txt)"
+        echo "issue=$issue" >> "$GITHUB_OUTPUT"
+
+    - name: Post Comment on GitHub Issue
+      if: steps.issue-id.outputs.issue != ''
+      working-directory: firebase-dataconnect/ci
+      shell: bash
+      run: |
+        args=(
+          python
+          post_comment_for_job_results.py
+          --github-issue='${{ steps.issue-id.outputs.issue }}'
+          --github-workflow='${{ github.workflow }}'
+          --github-repository='${{ github.repository }}'
+          --github-ref='${{ github.ref }}'
+          --github-event-name='${{ github.event_name }}'
+          --github-sha='${{ github.sha }}'
+          --github-repository-html-url='${{ github.event.repository.html_url }}'
+          --github-run-id='${{ github.run_id }}'
+          --github-run-number='${{ github.run_number }}'
+          --github-run-attempt='${{ github.run_attempt }}'
+        )
+
+        while read -r line; do
+          args=("${args[@]}" "$line")
+        done <'${{ inputs.job-results-file }}'
+
+        echo "${args[*]}"
+        exec "${args[@]}"

.github/workflows/check-vertexai-responses.yml renamed to .github/workflows/check-firebaseai-responses.yml

@@ -1,4 +1,4 @@
-name: Check Vertex AI Responses
+name: Check Firebase AI Responses
 
 on: pull_request
 
@@ -10,30 +10,30 @@ jobs:
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Clone mock responses
-        run: firebase-vertexai/update_responses.sh
+        run: firebase-ai/update_responses.sh
       - name: Find cloned and latest versions
         run: |
           CLONED=$(git describe --tags)
           LATEST=$(git tag --sort=v:refname | tail -n1)
           echo "cloned_tag=$CLONED" >> $GITHUB_ENV
           echo "latest_tag=$LATEST" >> $GITHUB_ENV
-        working-directory: firebase-vertexai/src/test/resources/vertexai-sdk-test-data
+        working-directory: firebase-ai/src/test/resources/vertexai-sdk-test-data
       - name: Find comment from previous run if exists
         uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
         id: fc
         with:
           issue-number: ${{github.event.number}}
-          body-includes: Vertex AI Mock Responses Check
+          body-includes: Firebase AI Mock Responses Check
       - name: Comment on PR if newer version is available
         if: ${{env.cloned_tag != env.latest_tag && !steps.fc.outputs.comment-id}}
         uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
         with:
           issue-number: ${{github.event.number}}
           body: >
-            ### Vertex AI Mock Responses Check :warning:
+            ### Firebase AI Mock Responses Check :warning:
 
-            A newer major version of the mock responses for Vertex AI unit tests is available.
-            [update_responses.sh](https://github.com/firebase/firebase-android-sdk/blob/main/firebase-vertexai/update_responses.sh)
+            A newer major version of the mock responses for Firebase AI unit tests is available.
+            [update_responses.sh](https://github.com/firebase/firebase-android-sdk/blob/main/firebase-ai/update_responses.sh)
             should be updated to clone the latest version of the responses: `${{env.latest_tag}}`
       - name: Delete comment when version gets updated
         if: ${{env.cloned_tag == env.latest_tag && steps.fc.outputs.comment-id}}

.github/workflows/ci_tests.yml

@@ -56,10 +56,14 @@ jobs:
           distribution: temurin
           cache: gradle
 
-      - name: Clone mock responses
+      - name: Clone vertexai mock responses
         if: matrix.module == ':firebase-vertexai'
         run: firebase-vertexai/update_responses.sh
 
+      - name: Clone ai mock responses
+        if: matrix.module == ':firebase-ai'
+        run: firebase-ai/update_responses.sh
+
       - name: Add google-services.json
         env:
           INTEG_TESTS_GOOGLE_SERVICES: ${{ secrets.INTEG_TESTS_GOOGLE_SERVICES }}

.github/workflows/dataconnect.yml

@@ -9,6 +9,7 @@ on:
       firebaseToolsVersion:
       gradleInfoLog:
         type: boolean
+      pythonVersion:
   pull_request:
     paths:
       - .github/workflows/dataconnect.yml
@@ -24,9 +25,10 @@ env:
   FDC_JAVA_VERSION: ${{ inputs.javaVersion || '17' }}
   FDC_ANDROID_EMULATOR_API_LEVEL: ${{ inputs.androidEmulatorApiLevel || '34' }}
   FDC_NODEJS_VERSION: ${{ inputs.nodeJsVersion || '20' }}
-  FDC_FIREBASE_TOOLS_VERSION: ${{ inputs.firebaseToolsVersion || '13.29.1' }}
+  FDC_FIREBASE_TOOLS_VERSION: ${{ inputs.firebaseToolsVersion || '14.2.0' }}
   FDC_FIREBASE_TOOLS_DIR: /tmp/firebase-tools
   FDC_FIREBASE_COMMAND: /tmp/firebase-tools/node_modules/.bin/firebase
+  FDC_PYTHON_VERSION: ${{ inputs.pythonVersion || '3.13' }}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -64,6 +66,12 @@ jobs:
         with:
           node-version: ${{ env.FDC_NODEJS_VERSION }}
 
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: ${{ env.FDC_PYTHON_VERSION }}
+
+      - run: pip install -r firebase-dataconnect/ci/requirements.txt
+
       - name: Install Firebase Tools ("firebase" command-line tool)
         run: |
           set -euo pipefail
@@ -74,7 +82,6 @@ jobs:
           npm install --fund=false --audit=false --save --save-exact firebase-tools@${{ env.FDC_FIREBASE_TOOLS_VERSION }}
 
       - name: Restore Gradle Cache
-        id: restore-gradle-cache
         uses: actions/cache/restore@d4323d4df104b026a6aa633fdb11d772146be0bf # 4.2.2
         if: github.event_name != 'schedule'
         with:
@@ -170,43 +177,28 @@ jobs:
             ~/.android/adb*
           key: avd-cache-zhdsn586je-api${{ env.FDC_ANDROID_EMULATOR_API_LEVEL }}-${{ github.run_id }}
 
-      - name: Data Connect Emulator Start
-        id: data-connect-emulator
+      - name: Start Firebase Emulators
         run: |
-          set -euo pipefail
-          set -xv
-
-          echo 'emulator.postgresConnectionUrl=postgresql://postgres:[email protected]:5432?sslmode=disable' > firebase-dataconnect/dataconnect.local.properties
-
-          ./gradlew \
-            ${{ (inputs.gradleInfoLog && '--info') || '' }} \
-            :firebase-dataconnect:connectors:runDebugDataConnectEmulator \
-            >firebase.emulator.dataconnect.log 2>&1 &
+          set -xveuo pipefail
 
-          echo "FIREBASE_DATA_CONNECT_EMULATOR_PID=$!" >> "$GITHUB_ENV"
-
-      - name: Firebase Auth Emulator Start
-        id: firebase-auth-emulator
-        run: |
-          set -euo pipefail
-          set -xv
+          # Use the same dataconnect binary as was used for code generation in gradle assemble
+          DATACONNECT_EMULATOR_BINARY_PATH="$(find "$PWD"/firebase-dataconnect/connectors/build/intermediates/dataconnect/debug/executable -type f)"
+          if [[ -z $DATACONNECT_EMULATOR_BINARY_PATH ]] ; then
+            echo "INTERNAL ERROR v7kg2dfhbc: unable to find data connect binary" >&2
+            exit 1
+          fi
+          export DATACONNECT_EMULATOR_BINARY_PATH
 
+          export FIREBASE_DATACONNECT_POSTGRESQL_STRING='postgresql://postgres:[email protected]:5432?sslmode=disable'
           cd firebase-dataconnect/emulator
-          ${{ env.FDC_FIREBASE_COMMAND }} emulators:start --only=auth >firebase.emulator.auth.log 2>&1 &
-
-          echo "FIREBASE_AUTH_EMULATOR_PID=$!" >> "$GITHUB_ENV"
+          ${{ env.FDC_FIREBASE_COMMAND }} emulators:start --only=auth,dataconnect >firebase.emulators.log 2>&1 &
 
-      - name: Logcat Log Capture Start
-        id: logcat-capture
+      - name: Start Logcat Capture
         continue-on-error: true
         run: |
-          set -euo pipefail
-          set -xv
-
+          set -xveuo pipefail
           "$ANDROID_HOME/platform-tools/adb" logcat >logcat.log 2>&1 &
 
-          echo "LOGCAT_PID=$!" >> "$GITHUB_ENV"
-
       - name: Gradle connectedCheck
         id: connectedCheck
         uses: reactivecircus/android-emulator-runner@62dbb605bba737720e10b196cb4220d374026a6d #v2.33.0
@@ -243,6 +235,15 @@ jobs:
         if: steps.connectedCheck.outcome != 'success'
         run: |
           set -euo pipefail
+
+          if [[ ! -e logcat.log ]] ; then
+            echo "WARNING dsdta43sxk: logcat log file not found; skipping scanning for test failures" >&2
+          else
+            echo "Scanning logcat output for failure details"
+            python firebase-dataconnect/ci/logcat_error_report.py --logcat-file=logcat.log
+            echo
+          fi
+
           echo 'Failing because the outcome of the "Gradle connectedCheck" step ("${{ steps.connectedCheck.outcome }}") was not successful'
           exit 1
 
@@ -256,30 +257,121 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           show-progress: false
+          sparse-checkout: '.github/'
       - uses: docker://rhysd/actionlint:1.7.7
         with:
           args: -color /github/workspace/.github/workflows/dataconnect.yml
 
+  python-ci-unit-tests:
+    continue-on-error: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          show-progress: false
+          sparse-checkout: 'firebase-dataconnect/ci/'
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: ${{ env.FDC_PYTHON_VERSION }}
+      - run: pip install -r firebase-dataconnect/ci/requirements.txt
+      - name: pytest
+        working-directory: firebase-dataconnect/ci
+        run: pytest --verbose --full-trace --color=no --strict-config
+
+  python-ci-lint:
+    continue-on-error: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          show-progress: false
+          sparse-checkout: 'firebase-dataconnect/ci/'
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: ${{ env.FDC_PYTHON_VERSION }}
+      - run: pip install -r firebase-dataconnect/ci/requirements.txt
+      - name: ruff check
+        working-directory: firebase-dataconnect/ci
+        run: ruff check --diff --verbose --no-cache --output-format=github --exit-non-zero-on-fix
+
+  python-ci-format:
+    continue-on-error: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          show-progress: false
+          sparse-checkout: 'firebase-dataconnect/ci/'
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: ${{ env.FDC_PYTHON_VERSION }}
+      - run: pip install -r firebase-dataconnect/ci/requirements.txt
+      - name: ruff format
+        working-directory: firebase-dataconnect/ci
+        run: ruff format --diff --verbose --no-cache
+
+  python-ci-type-check:
+    continue-on-error: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          show-progress: false
+          sparse-checkout: 'firebase-dataconnect/ci/'
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: ${{ env.FDC_PYTHON_VERSION }}
+      - run: pip install -r firebase-dataconnect/ci/requirements.txt
+      - name: pyright
+        working-directory: firebase-dataconnect/ci
+        run: pyright --warnings --stats
+
+  # The "send-notifications" job adds a comment to GitHub Issue
+  # https://github.com/firebase/firebase-android-sdk/issues/6857 with the results of the scheduled
+  # nightly runs. Interested parties can then subscribe to that issue to be aprised of the outcome
+  # of the nightly runs.
+  #
+  # When testing the comment-adding logic itself, you can add the line
+  #   trksmnkncd_notification_issue=6863
+  # into the PR's description to instead post a comment to issue #6863, an issue specifically
+  # created for testing, avoiding spamming the main issue to which others are subscribed.
   send-notifications:
-    needs: [integration-test, actionlint-dataconnect-yml]
+    needs:
+      - 'integration-test'
+      - 'actionlint-dataconnect-yml'
+      - 'python-ci-unit-tests'
+      - 'python-ci-lint'
+      - 'python-ci-format'
+      - 'python-ci-type-check'
     if: always()
     permissions:
       issues: write
     runs-on: ubuntu-latest
     steps:
-      - name: Post Comment on Issue #6857
-        if: github.event_name == 'schedule'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          set -euo pipefail
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          show-progress: false
+          sparse-checkout: |
+            firebase-dataconnect/ci/
+            .github/
 
-          cat >message.txt <<EOF
-          Result of workflow ${{ github.workflow }} at ${{ github.ref }}: ${{ job.status }}
-          ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}
-          job=${{ github.job }} run_id=${{ github.run_id }} run_number=${{ github.run_number }} run_attempt=${{ github.run_attempt }}
+      - name: gh auth login
+        run: echo '${{ secrets.GITHUB_TOKEN }}' | gh auth login --with-token
+
+      - name: Create Job Results File
+        run: |
+          set -xveuo pipefail
+          cat >'${{ runner.temp }}/job_results.txt' <<EOF
+          integration-test:${{ needs.integration-test.result }}
+          actionlint-dataconnect-yml:${{ needs.actionlint-dataconnect-yml.result }}
+          python-ci-unit-tests:${{ needs.python-ci-unit-tests.result }}
+          python-ci-lint:${{ needs.python-ci-lint.result }}
+          python-ci-format:${{ needs.python-ci-format.result }}
+          python-ci-type-check:${{ needs.python-ci-type-check.result }}
           EOF
 
-          echo "Posting comment on GitHub Issue: https://github.com/firebase/firebase-android-sdk/issues/6857:"
-          cat message.txt
-          gh issue comment 6857 --body-file message.txt -R ${{ github.repository }}
+      - uses: ./.github/actions/dataconnect-send-notifications
+        with:
+          python-version: ${{ env.FDC_PYTHON_VERSION }}
+          github-issue-for-scheduled-runs: "6857"
+          job-results-file: ${{ runner.temp }}/job_results.txt