Skip to content

Commit 20ffce6

Browse files
committed
fix
1 parent 6dc3297 commit 20ffce6

File tree

4 files changed

+53
-35
lines changed

4 files changed

+53
-35
lines changed

ci/fireci/fireciplugins/macrobenchmark/commands.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,7 @@ def macrobenchmark():
5050
type=click.Path(dir_okay=True, resolve_path=True, path_type=Path),
5151
default='macrobenchmark-output.json',
5252
show_default=True,
53-
help='The file for saving macrobenchmark test output. If running locally, the file contains '
54-
'the directory name of local test reports. If running remotely, the file contains Firebase '
55-
'Test Lab results directory names.'
53+
help='The file for saving macrobenchmark test output if running on Firebase Test Lab.'
5654
)
5755
@ci_command(group=macrobenchmark)
5856
def run(build_only: bool, local: bool, repeat: int, output: Path):

ci/fireci/fireciplugins/macrobenchmark/run/runner.py

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
import json
1717
import logging
1818
import tempfile
19+
20+
import click
1921
import yaml
2022

2123
from .test_project_builder import TestProjectBuilder
@@ -34,25 +36,38 @@ async def start(build_only: bool, local: bool, repeat: int, output: Path):
3436
test_dir = _prepare_test_directory()
3537
template_project_dir = Path('health-metrics/benchmark/template')
3638

37-
test_projects = []
38-
for test_config in config['test-apps']:
39-
builder = TestProjectBuilder(test_config, test_dir, template_project_dir, product_versions)
40-
test_projects.append(builder.build())
39+
test_projects = [
40+
TestProjectBuilder(
41+
test_config,
42+
test_dir,
43+
template_project_dir,
44+
product_versions,
45+
).build() for test_config in config['test-apps']]
4146

4247
if not build_only:
4348
if local:
44-
test_outputs = []
4549
for test_project in test_projects:
46-
test_output = test_project.run_local(repeat)
47-
test_outputs.append(test_output)
50+
test_project.run_local(repeat)
4851
else:
49-
remote_runs = [x.run_remote(repeat) for x in test_projects]
50-
test_outputs = await asyncio.gather(*remote_runs)
52+
remote_runs = [test_project.run_remote(repeat) for test_project in test_projects]
53+
results = await asyncio.gather(*remote_runs, return_exceptions=True)
54+
test_outputs = [x for x in results if not isinstance(x, Exception)]
55+
exceptions = [x for x in results if isinstance(x, Exception)]
56+
57+
with open(output, 'w') as file:
58+
json.dump(test_outputs, file)
59+
logger.info(f'Output of remote testing saved to: {output}')
60+
61+
if exceptions:
62+
logger.error(f'Exceptions occurred: {exceptions}')
63+
for test_output in test_outputs:
64+
if test_output['exceptions']:
65+
logger.error(f'Exceptions occurred: {test_output["exceptions"]}')
5166

52-
with open(output, 'w') as file:
53-
json.dump(test_outputs, file)
67+
if exceptions or any(test_output['exceptions'] for test_output in test_outputs):
68+
raise click.ClickException('Macrobenchmark test failed with above exceptions')
5469

55-
logger.info(f'Completed macrobenchmark test and saved output to "{output}"')
70+
logger.info(f'Completed macrobenchmark test successfully')
5671

5772

5873
def _assemble_all_products() -> dict[str, str]:

ci/fireci/fireciplugins/macrobenchmark/run/test_project.py

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import glob
1717
import re
1818
import shutil
19+
import random
1920

2021
from .log_decorator import LogDecorator
2122
from .utils import execute, execute_async, generate_test_run_id
@@ -27,10 +28,10 @@
2728
logger = getLogger('fireci.macrobenchmark')
2829

2930

30-
class TestOutput(TypedDict, total=False):
31+
class RemoteTestOutput(TypedDict, total=False):
3132
project: str
32-
local_reports_dir: str
33-
ftl_results_dirs: list[str]
33+
successful_runs: list[str]
34+
exceptions: list[str] # Using str due to Exception being not JSON serializable
3435

3536

3637
class TestProject:
@@ -39,7 +40,7 @@ def __init__(self, name: str, project_dir: Path, custom_logger: Logger | LoggerA
3940
self.test_project_dir = project_dir
4041
self.logger = custom_logger
4142

42-
def run_local(self, repeat: int) -> TestOutput:
43+
def run_local(self, repeat: int):
4344
self.logger.info(f'Running test locally for {repeat} times ...')
4445
local_reports_dir = self.test_project_dir.joinpath('_reports')
4546

@@ -59,10 +60,9 @@ def run_local(self, repeat: int) -> TestOutput:
5960
shutil.copy(report, device_dir)
6061
run_logger.debug(f'Copied report file "{report}" to "{device_dir}"')
6162

62-
self.logger.info(f'Completed all {repeat} runs, reports saved at "{local_reports_dir}"')
63-
return TestOutput(project=self.name, local_reports_dir=str(local_reports_dir))
63+
self.logger.info(f'Finished all {repeat} runs, local reports dir: "{local_reports_dir}"')
6464

65-
async def run_remote(self, repeat: int) -> TestOutput:
65+
async def run_remote(self, repeat: int) -> RemoteTestOutput:
6666
self.logger.info(f'Running test remotely for {repeat} times ...')
6767

6868
with chdir(self.test_project_dir):
@@ -71,9 +71,9 @@ async def run_remote(self, repeat: int) -> TestOutput:
7171
test_apk_path = glob.glob('**/macrobenchmark-benchmark.apk', recursive=True)[0]
7272
self.logger.info(f'App apk: "{app_apk_path}", Test apk: "{test_apk_path}"')
7373

74-
async def run(index: int, results_dir: str):
74+
async def run(index: int, run_id: str) -> str:
7575
run_logger = LogDecorator(self.logger, f'run-{index}')
76-
run_logger.info(f'Run-{index}: {results_dir}')
76+
run_logger.info(f'Run-{index}: {run_id}')
7777
ftl_environment_variables = [
7878
'clearPackageData=true',
7979
'additionalTestOutputDir=/sdcard/Download',
@@ -84,19 +84,25 @@ async def run(index: int, results_dir: str):
8484
args += ['--type', 'instrumentation']
8585
args += ['--app', app_apk_path]
8686
args += ['--test', test_apk_path]
87-
args += ['--device', 'model=f2q,version=30,locale=en,orientation=portrait']
8887
args += ['--device', 'model=oriole,version=32,locale=en,orientation=portrait']
89-
args += ['--device', 'model=redfin,version=30,locale=en,orientation=portrait']
9088
args += ['--directories-to-pull', '/sdcard/Download']
9189
args += ['--results-bucket', 'fireescape-benchmark-results']
92-
args += ['--results-dir', results_dir]
90+
args += ['--results-dir', run_id]
9391
args += ['--environment-variables', ','.join(ftl_environment_variables)]
9492
args += ['--timeout', '30m']
9593
args += ['--project', 'fireescape-c4819']
9694
await execute_async(executable, *args, logger=run_logger)
95+
return run_id
96+
97+
runs = [run(i, generate_test_run_id()) for i in range(repeat)]
98+
results = await asyncio.gather(*runs, return_exceptions=True)
99+
successes = [x for x in results if not isinstance(x, Exception)]
100+
exceptions = [x for x in results if isinstance(x, Exception)]
101+
102+
self.logger.info(f'Finished all {repeat} runs, successes: {successes}, failures: {exceptions}')
97103

98-
ftl_results_dirs = [generate_test_run_id() for _ in range(repeat)]
99-
runs = [run(i, ftl_results_dirs[i]) for i in range(repeat)]
100-
await asyncio.gather(*runs)
101-
self.logger.info(f'Completed all {repeat} runs, ftl results dirs: {ftl_results_dirs}')
102-
return TestOutput(project=self.name, ftl_results_dirs=ftl_results_dirs)
104+
return RemoteTestOutput(
105+
project=self.name,
106+
successful_runs=successes,
107+
exceptions=[str(e) for e in exceptions]
108+
)

ci/fireci/fireciplugins/macrobenchmark/run/utils.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
import click
1615
import datetime
1716
import string
1817
import random
@@ -45,7 +44,7 @@ def execute(program: str, *args: [str], logger: Logger | LoggerAdapter) -> None:
4544
else:
4645
message = f'"{command}" failed with return code {popen.returncode}'
4746
logger.error(message)
48-
raise click.ClickException(message)
47+
raise RuntimeError(message)
4948

5049

5150
async def execute_async(program: str, *args: [str], logger: Logger | LoggerAdapter) -> None:
@@ -62,4 +61,4 @@ async def execute_async(program: str, *args: [str], logger: Logger | LoggerAdapt
6261
else:
6362
message = f'"{command}" failed with return code {process.returncode}'
6463
logger.error(message)
65-
raise click.ClickException(message)
64+
raise RuntimeError(message)

0 commit comments

Comments
 (0)