diff --git a/ci/fireci/fireciplugins/macrobenchmark/analyze/aggregator.py b/ci/fireci/fireciplugins/macrobenchmark/analyze/aggregator.py index 5b75e3f2678..8c52e90ca51 100644 --- a/ci/fireci/fireciplugins/macrobenchmark/analyze/aggregator.py +++ b/ci/fireci/fireciplugins/macrobenchmark/analyze/aggregator.py @@ -47,6 +47,7 @@ def mapper(quantile: float) -> str: return f'p{int(quantile * 100)}' # Optionally save percentiles and dispersions to file if output_dir: + output_dir.mkdir(parents=True, exist_ok=True) percentiles.to_json(output_dir.joinpath('percentiles.json'), orient='index') dispersions.to_json(output_dir.joinpath('dispersions.json'), orient='index') logger.info(f'Percentiles and dispersions saved in: {output_dir}') @@ -63,8 +64,8 @@ def calculate_statistic_diff( ): logger.info(f'Calculating statistic diff for trace "{trace}" on device "{device}" ...') - ctl_percentiles, _ = calculate_statistic(trace, device, control) - exp_percentiles, _ = calculate_statistic(trace, device, experimental) + ctl_percentiles, _ = calculate_statistic(trace, device, control, output_dir.joinpath("ctl")) + exp_percentiles, _ = calculate_statistic(trace, device, experimental, output_dir.joinpath("exp")) ctl_mean = ctl_percentiles.mean() exp_mean = exp_percentiles.mean() @@ -74,6 +75,7 @@ def calculate_statistic_diff( # Optionally save statistics to file if output_dir: + output_dir.mkdir(parents=True, exist_ok=True) delta.to_json(output_dir.joinpath('delta.json')) percentage.to_json(output_dir.joinpath('percentage.json')) logger.info(f'Percentiles diff saved in: {output_dir}') diff --git a/ci/fireci/fireciplugins/macrobenchmark/analyze/analyzer.py b/ci/fireci/fireciplugins/macrobenchmark/analyze/analyzer.py index 86b8ec4ca76..daf63b93996 100644 --- a/ci/fireci/fireciplugins/macrobenchmark/analyze/analyzer.py +++ b/ci/fireci/fireciplugins/macrobenchmark/analyze/analyzer.py @@ -65,7 +65,6 @@ def _process(data_points: List[DataPoint], output_dir: Path) -> None: with progressbar(trace_device_combinations) as combinations: for trace, device in combinations: combination_dir = output_dir.joinpath(trace, device) - combination_dir.mkdir(parents=True, exist_ok=True) subset = _filter_subset(data, trace, device) calculate_statistic(trace, device, subset, combination_dir) plot_graph(trace, device, subset, combination_dir) @@ -88,7 +87,6 @@ def _diff( with progressbar(trace_device_combinations) as combinations: for trace, device in combinations: combination_dir = output_dir.joinpath(trace, device) - combination_dir.mkdir(parents=True, exist_ok=True) ctl_subset = _filter_subset(ctl_data, trace, device) exp_subset = _filter_subset(exp_data, trace, device) diff --git a/ci/fireci/fireciplugins/macrobenchmark/analyze/plotter.py b/ci/fireci/fireciplugins/macrobenchmark/analyze/plotter.py index ac73f80815c..31c98c229c0 100644 --- a/ci/fireci/fireciplugins/macrobenchmark/analyze/plotter.py +++ b/ci/fireci/fireciplugins/macrobenchmark/analyze/plotter.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +import matplotlib.pyplot as plt import numpy as np import pandas as pd import seaborn as sns @@ -27,6 +28,8 @@ def plot_graph(trace: str, device: str, data: pd.DataFrame, output_dir: Path): logger.info(f'Plotting graphs for trace "{trace}" on device "{device}" ...') + output_dir.mkdir(parents=True, exist_ok=True) + unique_run_ids = len(data['run_id'].unique()) col_wrap = int(np.ceil(np.sqrt(unique_run_ids))) @@ -34,6 +37,7 @@ def plot_graph(trace: str, device: str, data: pd.DataFrame, output_dir: Path): histograms.set_axis_labels(x_var=f'{trace} (ms)') histograms.set_titles(f'{device} ({{col_var}} = {{col_name}})') histograms.savefig(output_dir.joinpath('histograms.svg')) + plt.close(histograms.fig) distributions = sns.displot( data=data, x='duration', kde=True, height=8, @@ -41,6 +45,7 @@ def plot_graph(trace: str, device: str, data: pd.DataFrame, output_dir: Path): ) distributions.set_axis_labels(x_var=f'{trace} (ms)').set(title=device) distributions.savefig(output_dir.joinpath('distributions.svg')) + plt.close(distributions.fig) logger.info(f'Graphs saved in: {output_dir}') @@ -54,6 +59,8 @@ def plot_diff_graph( ): logger.info(f'Plotting distribution diff graph for trace "{trace}" on device "{device}" ...') + output_dir.mkdir(parents=True, exist_ok=True) + control_run_ids = control['run_id'] experimental_run_ids = experimental['run_id'] all_data = pd.concat([control, experimental]) @@ -66,5 +73,6 @@ def plot_diff_graph( ) distribution_diff.set_axis_labels(x_var=f'{trace} (ms)').set(title=device) distribution_diff.savefig(output_dir.joinpath('distribution_diff.svg')) + plt.close(distribution_diff.fig) logger.info(f'Graph saved in: {output_dir}') diff --git a/ci/fireci/fireciplugins/macrobenchmark/run/test_project.py b/ci/fireci/fireciplugins/macrobenchmark/run/test_project.py index 9a7bb22befd..24eebffb18f 100644 --- a/ci/fireci/fireciplugins/macrobenchmark/run/test_project.py +++ b/ci/fireci/fireciplugins/macrobenchmark/run/test_project.py @@ -83,7 +83,7 @@ async def run(index: int, run_id: str) -> str: args += ['--type', 'instrumentation'] args += ['--app', app_apk_path] args += ['--test', test_apk_path] - args += ['--device', 'model=oriole,version=32,locale=en,orientation=portrait'] + args += ['--device', 'model=redfin,version=30,locale=en,orientation=portrait'] args += ['--directories-to-pull', '/sdcard/Download'] args += ['--results-bucket', 'fireescape-benchmark-results'] args += ['--results-dir', run_id] diff --git a/ci/fireci/setup.cfg b/ci/fireci/setup.cfg index 1237d1a6af9..466898d3cb6 100644 --- a/ci/fireci/setup.cfg +++ b/ci/fireci/setup.cfg @@ -28,6 +28,10 @@ console_scripts = strict_optional = False [mypy-google.cloud] ignore_missing_imports = True +[mypy-matplotlib] +ignore_missing_imports = True +[mypy-matplotlib.pyplot] +ignore_missing_imports = True [mypy-pandas] ignore_missing_imports = True [mypy-pystache]