Refine startup time test app

- Forece initialize all components
- Include fire-perf-early in measurements
- Test only changed products in pull requests

Author: yifanyang

.github/workflows/health-metrics.yml

@@ -122,7 +122,12 @@ jobs:
           echo $INTEG_TESTS_GOOGLE_SERVICES | base64 -d > $BENCHMARK_APP_LOCATION
       - name: Run startup-time tests (presubmit)
         if: ${{ github.event_name == 'pull_request' }}
-        run: fireci macrobenchmark ci --pull-request
+        run: |
+          git diff --name-only HEAD~1 | \
+            xargs printf -- '--changed-git-paths %s\n' | \
+            xargs ./gradlew writeChangedProjects --output-file-path=modules.json
+          fireci macrobenchmark ci --pull-request --changed-modules-file modules.json
       - name: Run startup-time tests (post-submit)
         if: ${{ github.event_name == 'push' }}
-        run: fireci macrobenchmark ci --push
+        run: |
+          fireci macrobenchmark ci --push

ci/fireci/fireciplugins/macrobenchmark/commands.py

@@ -15,13 +15,16 @@
 import asyncio
 import click
 import json
+import logging
 
 from .analyze import analyzer
 from .run import runner
 from fireci import ci_command, ci_utils, uploader
 from pathlib import Path
 from typing import List
 
+logger = logging.getLogger('fireci.macrobenchmark')
+
 
 @ci_command(cls=click.Group)
 def macrobenchmark():
@@ -38,7 +41,7 @@ def macrobenchmark():
 )
 @click.option(
   '--local/--remote',
-  required=True,
+  default=True,
   help='Run the test on local devices or Firebase Test Lab.'
 )
 @click.option(
@@ -132,28 +135,45 @@ def analyze(
   required=True,
   help='Whether the test is running for a pull request or a push event.'
 )
+@click.option(
+  '--changed-modules-file',
+  type=click.Path(resolve_path=True, path_type=Path),
+  help='Contains a list of changed modules in the current pull request.'
+)
 @click.option(
   '--repeat',
   default=10,
   show_default=True,
   help='Number of times to repeat the test (for obtaining more data points).'
 )
 @ci_command(group=macrobenchmark)
-def ci(pull_request, repeat):
+def ci(pull_request: bool, changed_modules_file: Path, repeat: int):
   """Run tests in CI and upload results to the metric service."""
 
-  # TODO(yifany): run tests only for affected product in pull requests
-
   output_path = Path("macrobenchmark-test-output.json")
   exception = None
+
   try:
-    asyncio.run(runner.start(build_only=False, local=False, repeat=repeat, output=output_path))
+    if pull_request:
+      asyncio.run(
+        runner.start(
+          build_only=False,
+          local=False,
+          repeat=repeat,
+          output=output_path,
+          changed_modules_file=changed_modules_file,
+        )
+      )
+    else:
+      asyncio.run(runner.start(build_only=False, local=False, repeat=repeat, output=output_path))
   except Exception as e:
+    logger.error(f"Error: {e}")
     exception = e
 
   with open(output_path) as output_file:
     output = json.load(output_file)
-    ftl_dirs = list(filter(lambda x: x['project'] == 'all-included', output))[0]['successful_runs']
+    project_name = 'test-changed' if pull_request else 'test-all'
+    ftl_dirs = list(filter(lambda x: x['project'] == project_name, output))[0]['successful_runs']
     ftl_bucket_name = 'fireescape-benchmark-results'
 
     log = ci_utils.ci_log_link()

ci/fireci/fireciplugins/macrobenchmark/run/runner.py

@@ -22,18 +22,25 @@
 from .test_project_builder import TestProjectBuilder
 from .utils import execute
 from pathlib import Path
-from typing import Dict
+from typing import Dict, List
 
 
 logger = logging.getLogger('fireci.macrobenchmark')
 
 
-async def start(build_only: bool, local: bool, repeat: int, output: Path):
+async def start(
+    build_only: bool,
+    local: bool,
+    repeat: int,
+    output: Path,
+    changed_modules_file: Path = None
+):
   logger.info('Starting macrobenchmark test ...')
 
   config = _process_config_yaml()
   product_versions = _assemble_all_products()
   test_dir = _prepare_test_directory()
+  changed_modules = _process_changed_modules(changed_modules_file)
   template_project_dir = Path('health-metrics/benchmark/template')
 
   test_projects = [
@@ -42,6 +49,7 @@ async def start(build_only: bool, local: bool, repeat: int, output: Path):
       test_dir,
       template_project_dir,
       product_versions,
+      changed_modules,
     ).build() for test_config in config['test-apps']]
 
   if not build_only:
@@ -99,3 +107,17 @@ def _prepare_test_directory() -> Path:
   test_dir = tempfile.mkdtemp(prefix='benchmark-test-')
   logger.info(f'Temporary test directory created at: {test_dir}')
   return Path(test_dir)
+
+
+def _process_changed_modules(path: Path) -> List[str]:
+  results = []
+  if path:
+    with open(path) as changed_modules_file:
+      changed_modules = json.load(changed_modules_file)
+      for module in changed_modules:
+        names = module.split(':')
+        for name in names:
+          if name.startswith('firebase-'):
+            results.append(f'com.google.firebase:{name}')
+  logger.info(f"Extracted changed modules {results} from {path}")
+  return results

ci/fireci/fireciplugins/macrobenchmark/run/test_project_builder.py

@@ -21,7 +21,7 @@
 from .test_project import TestProject
 from .utils import execute
 from pathlib import Path
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 
 logger = logging.getLogger('fireci.macrobenchmark')
@@ -33,13 +33,15 @@ def __init__(
       test_config: Any,
       test_dir: Path,
       template_project_dir: Path,
-      product_versions: Dict[str, str]
+      product_versions: Dict[str, str],
+      changed_modules: List[str],
   ):
     self.test_config = test_config
     self.template_project_dir = template_project_dir
     self.product_versions = product_versions
+    self.changed_modules = changed_modules
 
-    self.name = test_config['name']
+    self.name = 'test-changed' if changed_modules else 'test-all'
     self.logger = LogDecorator(logger, self.name)
     self.project_dir = test_dir.joinpath(self.name)
 
@@ -77,7 +79,8 @@ def _flesh_out_mustache_template_files(self):
           dependency = {'key': key, 'version': version}
         else:
           dependency = {'key': dep, 'version': self.product_versions[dep]}
-        mustache_context['dependencies'].append(dependency)
+        if not self.changed_modules or dep in self.changed_modules:
+          mustache_context['dependencies'].append(dependency)
 
     renderer = pystache.Renderer()
     mustaches = self.project_dir.rglob('**/*.mustache')

firebase-common/src/main/java/com/google/firebase/FirebaseApp.java

@@ -472,7 +472,7 @@ public boolean isDefaultApp() {
   /** @hide */
   @VisibleForTesting
   @RestrictTo(Scope.TESTS)
-  void initializeAllComponents() {
+  public void initializeAllComponents() {
     componentRuntime.initializeAllComponentsForTests();
   }
 

health-metrics/benchmark/config.yaml

@@ -56,6 +56,7 @@ test-apps:
       - fire-installations
       - firebase-ml-modeldownloader
       - fire-perf
+      - fire-perf-early
       - fire-rc
       - fire-rtdb
       - fire-transport

health-metrics/benchmark/template/app/src/main/java/com/google/firebase/benchmark/MainActivity.kt

@@ -16,10 +16,13 @@ package com.google.firebase.benchmark
 
 import androidx.appcompat.app.AppCompatActivity
 import android.os.Bundle
+import com.google.firebase.FirebaseApp
 
 class MainActivity : AppCompatActivity() {
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
         setContentView(R.layout.activity_main)
+
+        FirebaseApp.getInstance().initializeAllComponents()
     }
 }