In the Test Summary Report, add "x/y" which means "x" out of "y" configs has errors. (#812)

Author: sunmou99

.github/workflows/integration_tests.yml

@@ -41,10 +41,6 @@ env:
   triggerLabelPrefix: "tests-requested: "
   triggerLabelFull: "tests-requested: full"
   triggerLabelQuick: "tests-requested: quick"
-  statusLabelInProgress: "tests: in-progress"
-  statusLabelFailed: "tests: failed"
-  statusLabelSucceeded: "tests: succeeded"
-  statusCommentIdentifier: "integration-test-status-comment"
   pythonVersion: '3.7'
   artifactRetentionDays: 2
   GITHUB_TOKEN: ${{ github.token }}

scripts/gha/summarize_test_results.py

@@ -208,8 +208,10 @@ def summarize_logs(dir, markdown=False, github_log=False):
   #               {"test": {"errors": [configs]},
   #                        {"failures": {failed_test: [configs]}},
   #                        {"flakiness": {flaky_test: [configs]}}}}
+  all_tested_configs = { "build_configs": [], "test_configs": []}
   for build_log_file in build_log_files:
     configs = get_configs_from_file_name(build_log_file, build_log_name_re)
+    all_tested_configs["build_configs"].append(configs)
     with open(build_log_file, "r") as log_reader:
       log_text = log_reader.read()
       if "__SUMMARY_MISSING__" in log_text:
@@ -223,6 +225,7 @@ def summarize_logs(dir, markdown=False, github_log=False):
 
   for test_log_file in test_log_files:
     configs = get_configs_from_file_name(test_log_file, test_log_name_re)
+    all_tested_configs["test_configs"].append(configs)
     with open(test_log_file, "r") as log_reader:
       log_text = log_reader.read()
       if "__SUMMARY_MISSING__" in log_text:
@@ -251,7 +254,9 @@ def summarize_logs(dir, markdown=False, github_log=False):
   # if failures (include flakiness) exist:
   # log_results format:
   #   { testapps: {configs: [failed tests]} }
-  log_results = reorganize_log(log_data)
+  all_tested_configs = reorganize_all_tested_configs(all_tested_configs)
+  logging.info("all_tested_configs: %s", all_tested_configs)
+  log_results = reorganize_log(log_data, all_tested_configs)
   log_lines = []
   if markdown:
     log_lines = print_markdown_table(log_results)
@@ -276,33 +281,44 @@ def get_configs_from_file_name(file_name, file_name_re):
   if "desktop" in configs: configs.remove("desktop")
   return configs
 
-def reorganize_log(log_data):
+
+def reorganize_all_tested_configs(tested_configs):
+  build_configs = reorganize_configs(tested_configs["build_configs"])
+  test_configs = reorganize_configs(tested_configs["test_configs"])
+  return { "build_configs": build_configs, "test_configs": test_configs}
+
+
+def reorganize_log(log_data, all_tested_configs):
   log_results = {}
   for (testapp, errors) in log_data.items():
     if errors.get("build"):
-      combined_configs = combine_configs(errors.get("build"))
+      reorganized_configs = reorganize_configs(errors.get("build"))
+      combined_configs = combine_configs(reorganized_configs, all_tested_configs["build_configs"])
       for (platform, configs) in combined_configs.items():
         for config in configs:
           all_configs = [["BUILD"], ["ERROR"], [CAPITALIZATIONS[platform]]]
           all_configs.extend(config)
           log_results.setdefault(testapp, {}).setdefault(flat_config(all_configs), [])
 
     if errors.get("test",{}).get("errors"):
-      combined_configs = combine_configs(errors.get("test",{}).get("errors"))
+      reorganized_configs = reorganize_configs(errors.get("test",{}).get("errors"))
+      combined_configs = combine_configs(reorganized_configs, all_tested_configs["test_configs"])
       for (platform, configs) in combined_configs.items():
         for config in configs:
           all_configs = [["TEST"], ["ERROR"], [CAPITALIZATIONS[platform]]]
           all_configs.extend(config)
           log_results.setdefault(testapp, {}).setdefault(flat_config(all_configs), [])
     for (test, configs) in errors.get("test",{}).get("failures",{}).items():
-      combined_configs = combine_configs(configs)
+      reorganized_configs = reorganize_configs(configs)
+      combined_configs = combine_configs(reorganized_configs, all_tested_configs["test_configs"])
       for (platform, configs) in combined_configs.items():
         for config in configs:
           all_configs = [["TEST"], ["FAILURE"], [CAPITALIZATIONS[platform]]]
           all_configs.extend(config)
           log_results.setdefault(testapp, {}).setdefault(flat_config(all_configs), []).append(test)
     for (test, configs) in errors.get("test",{}).get("flakiness",{}).items():
-      combined_configs = combine_configs(configs)
+      reorganized_configs = reorganize_configs(configs)
+      combined_configs = combine_configs(reorganized_configs, all_tested_configs["test_configs"])
       for (platform, configs) in combined_configs.items():
         for config in configs:
           all_configs = [["TEST"], ["FLAKINESS"], [CAPITALIZATIONS[platform]]]
@@ -312,11 +328,11 @@ def reorganize_log(log_data):
   return log_results
 
 
-# Combine Config Lists
+# Reorganize Config Lists
 # e.g.
 #     [['macos', 'simulator_min'], ['macos', 'simulator_target']]
 #     -> [[['macos'], ['simulator_min', 'simulator_target']]]
-def combine_configs(configs):
+def reorganize_configs(configs):
   platform_configs = {}
   for config in configs:
     platform = config[0]
@@ -339,47 +355,58 @@ def combine_configs(configs):
             if equals(configs_i, configs_j):
               remove_configs.append(configs_list[j])
               configk.append(configs_list[j][k])
-        configk = combine_config(configk, platform, k)
+        # configk = combine_config(configk, platform, k)
         configs_list[i][k] = configk
       for config in remove_configs:
         configs_list.remove(config)
 
   return platform_configs
 
-
 # If possible, combine kth config to "All *"
 # e.g.
 #     ['ubuntu', 'windows', 'macos']
-#     -> ['All os']
-def combine_config(config, platform, k):
+#     -> ['All 3 os']
+#     ['ubuntu', 'windows']
+#     -> ['2/3 os: ubuntu,windows': ]
+def combine_configs(error_configs, all_configs):
+  for (platform, configs_list) in error_configs.items():
+    for i in range(len(configs_list)):
+      for j in range(len(configs_list[i])):
+        configs_list[i][j] = combine_config(platform, configs_list[i][j], all_configs[platform][0][j], j)
+  return error_configs
+
+
+def combine_config(platform, config, config_value, k):
+  config_before_combination = config.copy()
+  logging.info("platform: %s; config: %s; config_value: %s", platform, config, config_value)
   if k == 1 and platform in ("android", "ios", "tvos"):
     # config_name = test_device here
     k = -1
   config_name = BUILD_CONFIGS[platform][k]
-  if PARAMETERS["integration_tests"]["matrix"]["expanded"].get(config_name):
-    config_value = PARAMETERS["integration_tests"]["matrix"]["expanded"][config_name]
-  else:
-    config_value = PARAMETERS["integration_tests"]["matrix"][config_name]
+  # if certain config failed for all values, add message "All *"
   if len(config_value) > 1 and len(config) == len(config_value):
     config = ["All %d %s" % (len(config_value), config_name)]
   elif config_name == "ios_device":
     ftl_devices = set(filter(lambda device: TEST_DEVICES.get(device).get("type") in "real", config_value))
     simulators = set(filter(lambda device: TEST_DEVICES.get(device).get("type") in "virtual", config_value))
-    if ftl_devices.issubset(set(config)):
+    if len(ftl_devices) > 1 and ftl_devices.issubset(set(config)):
       config.insert(0, "All %d FTL Devices" % len(ftl_devices))
       config = [x for x in config if (x not in ftl_devices)]
-    elif simulators.issubset(set(config)):
+    if len(simulators) > 1 and simulators.issubset(set(config)):
       config.insert(0, "All %d Simulators" % len(simulators))
       config = [x for x in config if (x not in simulators)]
   elif config_name == "android_device":
     ftl_devices = set(filter(lambda device: TEST_DEVICES.get(device).get("type") in "real", config_value))
     emulators = set(filter(lambda device: TEST_DEVICES.get(device).get("type") in "virtual", config_value))
-    if ftl_devices.issubset(set(config)):
+    if len(ftl_devices) > 1 and ftl_devices.issubset(set(config)):
       config.insert(0, "All %d FTL Devices" % len(ftl_devices))
       config = [x for x in config if (x not in ftl_devices)]
-    elif emulators.issubset(set(config)):
+    if len(emulators) > 1 and emulators.issubset(set(config)):
       config.insert(0, "All %d Emulators" % len(emulators))
       config = [x for x in config if (x not in emulators)]
+  # if certain config failed for more than 1 value but not all, add message "x/y" which means "x" out of "y" configs has errors.
+  if len(config_value) > 1 and config_before_combination == config:
+    config = ["%d/%d %s: %s" % (len(config), len(config_value), config_name, flat_config(config))]
 
   return config