Merge 6cf560d into e461cb7

Author: yifanyang

.github/workflows/health-metrics.yml

@@ -1,14 +1,27 @@
 name: Health Metrics
 
-on: [ pull_request, push ]
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      # add other feature branches here
+  # TODO(yifany): support workflow_dispatch for metric tests (or only for startup time test)
 
 env:
   GITHUB_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
 
 jobs:
   coverage:
     name: Coverage
-    if: (github.repository == 'Firebase/firebase-android-sdk' && github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)
+    if: |
+      (github.event_name == 'push' && github.repository == 'firebase/firebase-android-sdk')
+        || (github.event_name == 'pull_request'
+              && github.event.pull_request.head.repo.full_name == github.repository)
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -40,7 +53,10 @@ jobs:
 
   size:
     name: Size
-    if: (github.repository == 'Firebase/firebase-android-sdk' && github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)
+    if: |
+      (github.event_name == 'push' && github.repository == 'firebase/firebase-android-sdk')
+        || (github.event_name == 'pull_request'
+              && github.event.pull_request.head.repo.full_name == github.repository)
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -69,3 +85,44 @@ jobs:
       - name: Run size tests (post-submit)
         if: ${{ github.event_name == 'push' }}
         run: fireci binary_size
+
+  startup_time:
+    name: Startup Time
+    if: |
+      (github.event_name == 'push' && github.repository == 'firebase/firebase-android-sdk')
+        || (github.event_name == 'pull_request'
+              && github.event.pull_request.head.repo.full_name == github.repository)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+          submodules: true
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        with:
+          java-version: 11
+          distribution: temurin
+          cache: gradle
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - uses: google-github-actions/auth@v0
+        with:
+          credentials_json: '${{ secrets.GCP_SERVICE_ACCOUNT }}'
+      - uses: google-github-actions/setup-gcloud@v0
+      - name: Set up fireci
+        run: pip3 install -e ci/fireci
+      - name: Add google-services.json
+        env:
+          INTEG_TESTS_GOOGLE_SERVICES: ${{ secrets.INTEG_TESTS_GOOGLE_SERVICES }}
+          BENCHMARK_APP_LOCATION: health-metrics/benchmark/template/app/google-services.json
+        run: |
+          echo $INTEG_TESTS_GOOGLE_SERVICES | base64 -d > $BENCHMARK_APP_LOCATION
+      - name: Run startup-time tests (presubmit)
+        if: ${{ github.event_name == 'pull_request' }}
+        run: fireci macrobenchmark ci --pull-request
+      - name: Run startup-time tests (post-submit)
+        if: ${{ github.event_name == 'push' }}
+        run: fireci macrobenchmark ci --push

ci/fireci/fireci/uploader.py

@@ -17,7 +17,6 @@
 import os
 import requests
 import subprocess
-import urllib.parse
 
 
 _logger = logging.getLogger('fireci.uploader')
@@ -62,6 +61,7 @@ def _construct_request_endpoint_for_github_actions(metric_type):
 
   return endpoint
 
+
 def _construct_request_endpoint_for_prow(metric_type):
   repo_owner = os.getenv('REPO_OWNER')
   repo_name = os.getenv('REPO_NAME')

ci/fireci/fireciplugins/macrobenchmark/commands.py

@@ -14,10 +14,11 @@
 
 import asyncio
 import click
+import json
 
 from .analyze import analyzer
 from .run import runner
-from fireci import ci_command
+from fireci import ci_command, ci_utils, uploader
 from pathlib import Path
 from typing import List
 
@@ -125,4 +126,46 @@ def analyze(
     output_dir,
   )
 
+
+@click.option(
+  '--pull-request/--push',
+  required=True,
+  help='Whether the test is running for a pull request or a push event.'
+)
+@click.option(
+  '--repeat',
+  default=10,
+  show_default=True,
+  help='Number of times to repeat the test (for obtaining more data points).'
+)
+@ci_command(group=macrobenchmark)
+def ci(pull_request, repeat):
+  """Run tests in CI and upload results to the metric service."""
+
+  # TODO(yifany): run tests only for affected product in pull requests
+
+  output_path = Path("macrobenchmark-test-output.json")
+  exception = None
+  try:
+    asyncio.run(runner.start(build_only=False, local=False, repeat=repeat, output=output_path))
+  except Exception as e:
+    exception = e
+
+  with open(output_path) as output_file:
+    output = json.load(output_file)
+    ftl_dirs = list(filter(lambda x: x['project'] == 'all-included', output))[0]['successful_runs']
+    ftl_bucket_name = 'fireescape-benchmark-results'
+
+    log = ci_utils.ci_log_link()
+    ftl_results = list(map(lambda x: {'bucket': ftl_bucket_name, 'dir': x}, ftl_dirs))
+    startup_time_data = {'log': log, 'ftlResults': ftl_results}
+
+    if ftl_results:
+      metric_service_url = 'https://api.firebase-sdk-health-metrics.com'
+      access_token = ci_utils.gcloud_identity_token()
+      uploader.post_report(startup_time_data, metric_service_url, access_token, 'startup-time')
+
+  if exception:
+    raise exception
+
 # TODO(yifany): support of command chaining

ci/fireci/fireciplugins/macrobenchmark/run/test_project.py

@@ -83,12 +83,13 @@ async def run(index: int, run_id: str) -> str:
         args += ['--type', 'instrumentation']
         args += ['--app', app_apk_path]
         args += ['--test', test_apk_path]
+        args += ['--device', 'model=oriole,version=32,locale=en,orientation=portrait']
         args += ['--device', 'model=redfin,version=30,locale=en,orientation=portrait']
         args += ['--directories-to-pull', '/sdcard/Download']
         args += ['--results-bucket', 'fireescape-benchmark-results']
         args += ['--results-dir', run_id]
         args += ['--environment-variables', ','.join(ftl_environment_variables)]
-        args += ['--timeout', '30m']
+        args += ['--timeout', '45m']
         args += ['--project', 'fireescape-c4819']
         await execute_async(executable, *args, logger=run_logger)
         return run_id