Fetch and parse macrobenchmark results from GCS. (#2673)

Author: yifanyang

ci/fireci/fireciplugins/macrobenchmark.py

@@ -18,12 +18,16 @@
 import logging
 import os
 import random
+import re
 import shutil
+import statistics
 import sys
+import uuid
 
 import click
 import pystache
 import yaml
+from google.cloud import storage
 
 from fireci import ci_command
 from fireci.dir_utils import chdir
@@ -114,14 +118,18 @@ def __init__(
     self.artifact_versions = artifact_versions
     self.logger = MacrobenchmarkLoggerAdapter(logger, sdk_name)
     self.test_app_dir = os.path.join('test-apps', test_app_config['name'])
+    self.test_results_bucket = 'fireescape-benchmark-results'
+    self.test_results_dir = str(uuid.uuid4())
+    self.gcs_client = storage.Client()
 
   async def run(self):
-    """Starts the workflow of src creation, apks assembling, and FTL testing in order."""
-    await self._create_test_src()
-    await self._assemble_apks()
-    await self._upload_apks_to_ftl()
+    """Starts the workflow of src creation, apks assembly, FTL testing and results upload."""
+    await self._create_benchmark_projects()
+    await self._assemble_benchmark_apks()
+    await self._execute_benchmark_tests()
+    await self._upload_benchmark_results()
 
-  async def _create_test_src(self):
+  async def _create_benchmark_projects(self):
     app_name = self.test_app_config['name']
     app_id = self.test_app_config['application-id']
     self.logger.info(f'Creating test app "{app_name}" with application-id "{app_id}"...')
@@ -138,12 +146,12 @@ async def _create_test_src(self):
         with open(original_name, 'w') as file:
           file.write(result)
 
-  async def _assemble_apks(self):
+  async def _assemble_benchmark_apks(self):
     executable = './gradlew'
     args = ['assemble', 'assembleAndroidTest', '--project-dir', self.test_app_dir]
     await self._exec_subprocess(executable, args)
 
-  async def _upload_apks_to_ftl(self):
+  async def _execute_benchmark_tests(self):
     app_apk_path = glob.glob(f'{self.test_app_dir}/app/**/*.apk', recursive=True)[0]
     test_apk_path = glob.glob(f'{self.test_app_dir}/benchmark/**/*.apk', recursive=True)[0]
 
@@ -162,7 +170,8 @@ async def _upload_apks_to_ftl(self):
     args += ['--test', test_apk_path]
     args += ['--device', 'model=flame,version=30,locale=en,orientation=portrait']
     args += ['--directories-to-pull', '/sdcard/Download']
-    args += ['--results-bucket', 'gs://fireescape-macrobenchmark']
+    args += ['--results-bucket', f'gs://{self.test_results_bucket}']
+    args += ['--results-dir', self.test_results_dir]
     args += ['--environment-variables', ','.join(ftl_environment_variables)]
     args += ['--timeout', '30m']
     args += ['--project', 'fireescape-c4819']
@@ -196,6 +205,32 @@ async def _prepare_mustache_context(self):
 
     return mustache_context
 
+  async def _upload_benchmark_results(self):
+    results = []
+    blobs = self.gcs_client.list_blobs(self.test_results_bucket, prefix=self.test_results_dir)
+    files = [x for x in blobs if re.search(r'artifacts/[^/]*\.json', x.name)]
+    for file in files:
+      device = re.search(r'([^/]*)/artifacts/', file.name).group(1)
+      benchmarks = json.loads(file.download_as_bytes())['benchmarks']
+      for benchmark in benchmarks:
+        method = benchmark['name']
+        clazz = benchmark['className'].split('.')[-1]
+        runs = benchmark['metrics']['startupMs']['runs']
+        results.append({
+          'sdk': self.sdk_name,
+          'device': device,
+          'name': f'{clazz}.{method}',
+          'min': min(runs),
+          'max': max(runs),
+          'mean': statistics.mean(runs),
+          'median': statistics.median(runs),
+          'stdev': statistics.stdev(runs),
+          'unit': 'ms',
+        })
+    self.logger.info(f'Benchmark results: {results}')
+
+    # TODO(yifany): upload to metric service once it is ready
+
   async def _exec_subprocess(self, executable, args):
     command = " ".join([executable, *args])
     self.logger.info(f'Executing command: "{command}"...')

ci/fireci/setup.py

@@ -26,6 +26,7 @@
     version='0.1',
     install_requires=[
         'click==7.0',
+        'google-cloud-storage==1.38.0',
         'PyGithub==1.43.8',
         'pystache==0.5.4',
         'requests==2.23.0',