Improve fireci CLI for macrobenchmark (#4359)

- Output more analysis results
- Close figure after saving to file
- Switch to `redfin` for more consistent results

Author: yifanyang

ci/fireci/fireciplugins/macrobenchmark/analyze/aggregator.py

@@ -47,6 +47,7 @@ def mapper(quantile: float) -> str: return f'p{int(quantile * 100)}'
 
   # Optionally save percentiles and dispersions to file
   if output_dir:
+    output_dir.mkdir(parents=True, exist_ok=True)
     percentiles.to_json(output_dir.joinpath('percentiles.json'), orient='index')
     dispersions.to_json(output_dir.joinpath('dispersions.json'), orient='index')
     logger.info(f'Percentiles and dispersions saved in: {output_dir}')
@@ -63,8 +64,8 @@ def calculate_statistic_diff(
 ):
   logger.info(f'Calculating statistic diff for trace "{trace}" on device "{device}" ...')
 
-  ctl_percentiles, _ = calculate_statistic(trace, device, control)
-  exp_percentiles, _ = calculate_statistic(trace, device, experimental)
+  ctl_percentiles, _ = calculate_statistic(trace, device, control, output_dir.joinpath("ctl"))
+  exp_percentiles, _ = calculate_statistic(trace, device, experimental, output_dir.joinpath("exp"))
 
   ctl_mean = ctl_percentiles.mean()
   exp_mean = exp_percentiles.mean()
@@ -74,6 +75,7 @@ def calculate_statistic_diff(
 
   # Optionally save statistics to file
   if output_dir:
+    output_dir.mkdir(parents=True, exist_ok=True)
     delta.to_json(output_dir.joinpath('delta.json'))
     percentage.to_json(output_dir.joinpath('percentage.json'))
     logger.info(f'Percentiles diff saved in: {output_dir}')

ci/fireci/fireciplugins/macrobenchmark/analyze/analyzer.py

@@ -65,7 +65,6 @@ def _process(data_points: List[DataPoint], output_dir: Path) -> None:
   with progressbar(trace_device_combinations) as combinations:
     for trace, device in combinations:
       combination_dir = output_dir.joinpath(trace, device)
-      combination_dir.mkdir(parents=True, exist_ok=True)
       subset = _filter_subset(data, trace, device)
       calculate_statistic(trace, device, subset, combination_dir)
       plot_graph(trace, device, subset, combination_dir)
@@ -88,7 +87,6 @@ def _diff(
   with progressbar(trace_device_combinations) as combinations:
     for trace, device in combinations:
       combination_dir = output_dir.joinpath(trace, device)
-      combination_dir.mkdir(parents=True, exist_ok=True)
 
       ctl_subset = _filter_subset(ctl_data, trace, device)
       exp_subset = _filter_subset(exp_data, trace, device)

ci/fireci/fireciplugins/macrobenchmark/analyze/plotter.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import logging
+import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 import seaborn as sns
@@ -27,20 +28,24 @@
 def plot_graph(trace: str, device: str, data: pd.DataFrame, output_dir: Path):
   logger.info(f'Plotting graphs for trace "{trace}" on device "{device}" ...')
 
+  output_dir.mkdir(parents=True, exist_ok=True)
+
   unique_run_ids = len(data['run_id'].unique())
   col_wrap = int(np.ceil(np.sqrt(unique_run_ids)))
 
   histograms = sns.displot(data=data, x='duration', kde=True, col="run_id", col_wrap=col_wrap)
   histograms.set_axis_labels(x_var=f'{trace} (ms)')
   histograms.set_titles(f'{device} ({{col_var}} = {{col_name}})')
   histograms.savefig(output_dir.joinpath('histograms.svg'))
+  plt.close(histograms.fig)
 
   distributions = sns.displot(
     data=data, x='duration', kde=True, height=8,
     hue='run_id', palette='muted', multiple='dodge'
   )
   distributions.set_axis_labels(x_var=f'{trace} (ms)').set(title=device)
   distributions.savefig(output_dir.joinpath('distributions.svg'))
+  plt.close(distributions.fig)
 
   logger.info(f'Graphs saved in: {output_dir}')
 
@@ -54,6 +59,8 @@ def plot_diff_graph(
 ):
   logger.info(f'Plotting distribution diff graph for trace "{trace}" on device "{device}" ...')
 
+  output_dir.mkdir(parents=True, exist_ok=True)
+
   control_run_ids = control['run_id']
   experimental_run_ids = experimental['run_id']
   all_data = pd.concat([control, experimental])
@@ -66,5 +73,6 @@ def plot_diff_graph(
   )
   distribution_diff.set_axis_labels(x_var=f'{trace} (ms)').set(title=device)
   distribution_diff.savefig(output_dir.joinpath('distribution_diff.svg'))
+  plt.close(distribution_diff.fig)
 
   logger.info(f'Graph saved in: {output_dir}')

ci/fireci/fireciplugins/macrobenchmark/run/test_project.py

@@ -83,7 +83,7 @@ async def run(index: int, run_id: str) -> str:
         args += ['--type', 'instrumentation']
         args += ['--app', app_apk_path]
         args += ['--test', test_apk_path]
-        args += ['--device', 'model=oriole,version=32,locale=en,orientation=portrait']
+        args += ['--device', 'model=redfin,version=30,locale=en,orientation=portrait']
         args += ['--directories-to-pull', '/sdcard/Download']
         args += ['--results-bucket', 'fireescape-benchmark-results']
         args += ['--results-dir', run_id]

ci/fireci/setup.cfg

@@ -28,6 +28,10 @@ console_scripts =
 strict_optional = False
 [mypy-google.cloud]
 ignore_missing_imports = True
+[mypy-matplotlib]
+ignore_missing_imports = True
+[mypy-matplotlib.pyplot]
+ignore_missing_imports = True
 [mypy-pandas]
 ignore_missing_imports = True
 [mypy-pystache]