16
16
import glob
17
17
import re
18
18
import shutil
19
+ import random
19
20
20
21
from .log_decorator import LogDecorator
21
22
from .utils import execute , execute_async , generate_test_run_id
27
28
logger = getLogger ('fireci.macrobenchmark' )
28
29
29
30
30
- class TestOutput (TypedDict , total = False ):
31
+ class RemoteTestOutput (TypedDict , total = False ):
31
32
project : str
32
- local_reports_dir : str
33
- ftl_results_dirs : list [str ]
33
+ successful_runs : list [ str ]
34
+ exceptions : list [str ] # Using str due to Exception being not JSON serializable
34
35
35
36
36
37
class TestProject :
@@ -39,7 +40,7 @@ def __init__(self, name: str, project_dir: Path, custom_logger: Logger | LoggerA
39
40
self .test_project_dir = project_dir
40
41
self .logger = custom_logger
41
42
42
- def run_local (self , repeat : int ) -> TestOutput :
43
+ def run_local (self , repeat : int ):
43
44
self .logger .info (f'Running test locally for { repeat } times ...' )
44
45
local_reports_dir = self .test_project_dir .joinpath ('_reports' )
45
46
@@ -59,10 +60,9 @@ def run_local(self, repeat: int) -> TestOutput:
59
60
shutil .copy (report , device_dir )
60
61
run_logger .debug (f'Copied report file "{ report } " to "{ device_dir } "' )
61
62
62
- self .logger .info (f'Completed all { repeat } runs, reports saved at "{ local_reports_dir } "' )
63
- return TestOutput (project = self .name , local_reports_dir = str (local_reports_dir ))
63
+ self .logger .info (f'Finished all { repeat } runs, local reports dir: "{ local_reports_dir } "' )
64
64
65
- async def run_remote (self , repeat : int ) -> TestOutput :
65
+ async def run_remote (self , repeat : int ) -> RemoteTestOutput :
66
66
self .logger .info (f'Running test remotely for { repeat } times ...' )
67
67
68
68
with chdir (self .test_project_dir ):
@@ -71,9 +71,9 @@ async def run_remote(self, repeat: int) -> TestOutput:
71
71
test_apk_path = glob .glob ('**/macrobenchmark-benchmark.apk' , recursive = True )[0 ]
72
72
self .logger .info (f'App apk: "{ app_apk_path } ", Test apk: "{ test_apk_path } "' )
73
73
74
- async def run (index : int , results_dir : str ):
74
+ async def run (index : int , run_id : str ) -> str :
75
75
run_logger = LogDecorator (self .logger , f'run-{ index } ' )
76
- run_logger .info (f'Run-{ index } : { results_dir } ' )
76
+ run_logger .info (f'Run-{ index } : { run_id } ' )
77
77
ftl_environment_variables = [
78
78
'clearPackageData=true' ,
79
79
'additionalTestOutputDir=/sdcard/Download' ,
@@ -84,19 +84,25 @@ async def run(index: int, results_dir: str):
84
84
args += ['--type' , 'instrumentation' ]
85
85
args += ['--app' , app_apk_path ]
86
86
args += ['--test' , test_apk_path ]
87
- args += ['--device' , 'model=f2q,version=30,locale=en,orientation=portrait' ]
88
87
args += ['--device' , 'model=oriole,version=32,locale=en,orientation=portrait' ]
89
- args += ['--device' , 'model=redfin,version=30,locale=en,orientation=portrait' ]
90
88
args += ['--directories-to-pull' , '/sdcard/Download' ]
91
89
args += ['--results-bucket' , 'fireescape-benchmark-results' ]
92
- args += ['--results-dir' , results_dir ]
90
+ args += ['--results-dir' , run_id ]
93
91
args += ['--environment-variables' , ',' .join (ftl_environment_variables )]
94
92
args += ['--timeout' , '30m' ]
95
93
args += ['--project' , 'fireescape-c4819' ]
96
94
await execute_async (executable , * args , logger = run_logger )
95
+ return run_id
96
+
97
+ runs = [run (i , generate_test_run_id ()) for i in range (repeat )]
98
+ results = await asyncio .gather (* runs , return_exceptions = True )
99
+ successes = [x for x in results if not isinstance (x , Exception )]
100
+ exceptions = [x for x in results if isinstance (x , Exception )]
101
+
102
+ self .logger .info (f'Finished all { repeat } runs, successes: { successes } , failures: { exceptions } ' )
97
103
98
- ftl_results_dirs = [ generate_test_run_id () for _ in range ( repeat )]
99
- runs = [ run ( i , ftl_results_dirs [ i ]) for i in range ( repeat )]
100
- await asyncio . gather ( * runs )
101
- self . logger . info ( f'Completed all { repeat } runs, ftl results dirs: { ftl_results_dirs } ' )
102
- return TestOutput ( project = self . name , ftl_results_dirs = ftl_results_dirs )
104
+ return RemoteTestOutput (
105
+ project = self . name ,
106
+ successful_runs = successes ,
107
+ exceptions = [ str ( e ) for e in exceptions ]
108
+ )
0 commit comments