capture errors in test execution

Author: ThunderKey

python_tool_competition_2024/calculation/cli_runner.py

@@ -33,20 +33,32 @@
 
 @overload
 def run_command(
-    config: Config, command: _COMMAND, *args: str, capture: Literal[True]
+    config: Config,
+    command: _COMMAND,
+    *args: str,
+    capture: Literal[True],
+    show_output_on_error: bool = ...,
 ) -> str:
     ...
 
 
 @overload
 def run_command(
-    config: Config, command: _COMMAND, *args: str, capture: Literal[False] = ...
+    config: Config,
+    command: _COMMAND,
+    *args: str,
+    capture: Literal[False] = ...,
+    show_output_on_error: bool = ...,
 ) -> None:
     ...
 
 
 def run_command(
-    config: Config, command: _COMMAND, *args: str, capture: Literal[True, False] = False
+    config: Config,
+    command: _COMMAND,
+    *args: str,
+    capture: Literal[True, False] = False,
+    show_output_on_error: bool = True,
 ) -> None | str:
     """
     Run a command on the command line.
@@ -70,7 +82,8 @@ def run_command(
             with config.console.capture() as console_capture:
                 output = _run_command(config, command, args)
         except CommandFailedError:
-            config.console.out(console_capture.get(), highlight=False, end="")
+            if show_output_on_error:
+                config.console.out(console_capture.get(), highlight=False, end="")
             raise
     return output if capture else None
 

python_tool_competition_2024/calculation/coverage_caluclator.py

@@ -31,7 +31,11 @@
 
 from ..calculation.cli_runner import run_command
 from ..config import Config
-from ..errors import ConditionCoverageError, TargetNotFoundInCoveragesError
+from ..errors import (
+    CommandFailedError,
+    ConditionCoverageError,
+    TargetNotFoundInCoveragesError,
+)
 from ..results import RatioResult
 from ..target_finder import Target
 
@@ -53,20 +57,27 @@ def _generate_coverage_xml(target: Target, config: Config) -> Path:
     coverage_xml = config.coverages_dir / f"{target.source_module}.xml"
     coverage_xml.unlink(missing_ok=True)
     if target.test.exists():
-        run_command(
-            config,
-            "pytest",
-            str(target.test),
-            f"--cov={target.source_module}",
-            "--cov-branch",
-            "--cov-report",
-            f"xml:{coverage_xml}",
-            "--color=yes",
-            # reset options that are not desired
-            "--cov-fail-under=0",
-            "--override-ini=addopts=",
-            "--override-ini=cache_dir=.pytest_competition_cache",
-        )
+        try:
+            run_command(
+                config,
+                "pytest",
+                str(target.test),
+                f"--cov={target.source_module}",
+                "--cov-branch",
+                "--cov-report",
+                f"xml:{coverage_xml}",
+                "--color=yes",
+                # reset options that are not desired
+                "--cov-fail-under=0",
+                "--override-ini=addopts=",
+                "--override-ini=cache_dir=.pytest_competition_cache",
+                show_output_on_error=False,
+            )
+        except CommandFailedError:
+            msg = f"Could not run pytest for {target.source_module}."
+            if not config.show_commands:
+                msg = f"{msg} Add -vv to show the console output."
+            config.console.print(msg, style="red")
 
     # if the test was not generated or it does not import the source
     if not coverage_xml.exists():

python_tool_competition_2024/calculation/mutation_calculator/cosmic_ray_calculator.py

@@ -27,6 +27,7 @@
 import toml
 
 from ...config import Config
+from ...errors import CommandFailedError
 from ...results import RatioResult
 from ...target_finder import Target
 from ..cli_runner import run_command
@@ -46,10 +47,23 @@ def calculate_mutation(target: Target, config: Config) -> RatioResult:
     run_command(
         config, "cosmic-ray", "init", str(files.config_file), str(files.db_file)
     )
-    run_command(config, "cosmic-ray", "baseline", str(files.config_file))
-    run_command(
-        config, "cosmic-ray", "exec", str(files.config_file), str(files.db_file)
-    )
+    try:
+        run_command(
+            config,
+            "cosmic-ray",
+            "baseline",
+            str(files.config_file),
+            show_output_on_error=False,
+        )
+    except CommandFailedError:
+        msg = f"Could not run mutation testing for {target.source_module}."
+        if not config.show_commands:
+            msg = f"{msg} Add -vv to show the console output."
+        config.console.print(msg, style="red")
+    else:
+        run_command(
+            config, "cosmic-ray", "exec", str(files.config_file), str(files.db_file)
+        )
     return _gather_results(files, config)
 
 

python_tool_competition_2024/calculation/mutation_calculator/mutpy_calculator.py

@@ -22,6 +22,7 @@
 import re
 
 from ...config import Config
+from ...errors import CommandFailedError
 from ...results import RatioResult
 from ...target_finder import Target
 from ..cli_runner import run_command
@@ -31,12 +32,30 @@
 _KILLED_REGEX = re.compile(
     r"\A\s+- killed: (?P<number>\d+) \((?P<percentage>\d+\.\d+%)\)\Z"
 )
+_EMPTY_MODULE = "typing"
 
 
 def calculate_mutation(target: Target, config: Config) -> RatioResult:
     """Calculate mutation analysis using mutpy."""
-    test_module = target.test_module if target.test.exists() else "typing"
-    output = run_command(
+    test_module = target.test_module if target.test.exists() else _EMPTY_MODULE
+    try:
+        output = _run_mutpy(target, config, test_module)
+    except CommandFailedError:
+        if test_module == _EMPTY_MODULE:
+            raise
+        msg = f"Could not run mutation testing for {target.source_module}."
+        if not config.show_commands:
+            msg = f"{msg} Add -vv to show the console output."
+        config.console.print(msg, style="red")
+        output = _run_mutpy(target, config, _EMPTY_MODULE)
+    lines = tuple(output.splitlines())
+    total = int(find_matching_line(_TOTAL_REGEX, lines).group("number"))
+    killed = int(find_matching_line(_KILLED_REGEX, lines).group("number"))
+    return RatioResult(total, killed)
+
+
+def _run_mutpy(target: Target, config: Config, test_module: str) -> str:
+    return run_command(
         config,
         "mut.py",
         "--target",
@@ -46,8 +65,5 @@ def calculate_mutation(target: Target, config: Config) -> RatioResult:
         "--runner",
         "pytest",
         capture=True,
+        show_output_on_error=False,
     )
-    lines = tuple(output.splitlines())
-    total = int(find_matching_line(_TOTAL_REGEX, lines).group("number"))
-    killed = int(find_matching_line(_KILLED_REGEX, lines).group("number"))
-    return RatioResult(total, killed)

tests/calculation/mutation_calculator/test_cosmic_ray_calculator.py

@@ -8,6 +8,7 @@
     calculate_mutation,
 )
 from python_tool_competition_2024.config import Config
+from python_tool_competition_2024.errors import CommandFailedError
 from python_tool_competition_2024.results import RatioResult
 from python_tool_competition_2024.target_finder import find_targets
 
@@ -72,6 +73,99 @@ def test_cosmic_ray_calculator(tmp_path: Path) -> None:
         }
 
 
+def test_cosmic_ray_calculator_with_failing_baseline(tmp_path: Path) -> None:
+    with mock.patch(
+        "python_tool_competition_2024.calculation.mutation_calculator.cosmic_ray_calculator.run_command"
+    ) as run_command_mock:
+        run_command_mock.side_effect = _OutputCounter(fail_baseline=True)
+        mock.seal(run_command_mock)
+        generated_tests = tmp_path / "dummy" / "generated_tests"
+        generated_tests.mkdir(parents=True)
+        (generated_tests / "test_example1.py").touch()
+        (generated_tests / "test_example2.py").touch()
+        config = get_test_config(
+            show_commands=False,
+            show_failures=False,
+            targets_dir=TARGETS_DIR,
+            results_dir=tmp_path,
+        )
+        targets = find_targets(config)
+        with config.console.capture() as capture:
+            assert {
+                target.source_module: calculate_mutation(target, config)
+                for target in targets
+            } == {
+                "example1": RatioResult(10, 0),
+                "example2": RatioResult(11, 1),
+                "sub_example": RatioResult(12, 2),
+                "sub_example.example3": RatioResult(13, 3),
+            }
+        assert tuple(capture.get().splitlines()) == tuple(
+            (
+                f"Could not run mutation testing for {module}. "
+                "Add -vv to show the console output."
+            )
+            for module in (
+                "example1",
+                "example2",
+                "sub_example",
+                "sub_example.example3",
+            )
+        )
+
+        assert run_command_mock.call_args_list == [
+            *_cr_calls(config, "example1", skip_exec=True),
+            *_cr_calls(config, "example2", skip_exec=True),
+            *_cr_calls(config, "sub_example", skip_exec=True),
+            *_cr_calls(config, "sub_example.example3", skip_exec=True),
+        ]
+
+
+def test_cosmic_ray_calculator_with_failing_baseline_and_output(tmp_path: Path) -> None:
+    with mock.patch(
+        "python_tool_competition_2024.calculation.mutation_calculator.cosmic_ray_calculator.run_command"
+    ) as run_command_mock:
+        run_command_mock.side_effect = _OutputCounter(fail_baseline=True)
+        mock.seal(run_command_mock)
+        generated_tests = tmp_path / "dummy" / "generated_tests"
+        generated_tests.mkdir(parents=True)
+        (generated_tests / "test_example1.py").touch()
+        (generated_tests / "test_example2.py").touch()
+        config = get_test_config(
+            show_commands=True,
+            show_failures=False,
+            targets_dir=TARGETS_DIR,
+            results_dir=tmp_path,
+        )
+        targets = find_targets(config)
+        with config.console.capture() as capture:
+            assert {
+                target.source_module: calculate_mutation(target, config)
+                for target in targets
+            } == {
+                "example1": RatioResult(10, 0),
+                "example2": RatioResult(11, 1),
+                "sub_example": RatioResult(12, 2),
+                "sub_example.example3": RatioResult(13, 3),
+            }
+        assert tuple(capture.get().splitlines()) == tuple(
+            f"Could not run mutation testing for {module}."
+            for module in (
+                "example1",
+                "example2",
+                "sub_example",
+                "sub_example.example3",
+            )
+        )
+
+        assert run_command_mock.call_args_list == [
+            *_cr_calls(config, "example1", skip_exec=True),
+            *_cr_calls(config, "example2", skip_exec=True),
+            *_cr_calls(config, "sub_example", skip_exec=True),
+            *_cr_calls(config, "sub_example.example3", skip_exec=True),
+        ]
+
+
 def test_gather_results_not_started() -> None:
     with mock.patch(
         "python_tool_competition_2024.calculation.mutation_calculator.cosmic_ray_calculator.run_command"
@@ -107,16 +201,28 @@ def test_gather_results_not_completed() -> None:
         )
 
 
-def _cr_calls(config: Config, target: str) -> tuple[mock._Call, ...]:
+def _cr_calls(
+    config: Config, target: str, *, skip_exec: bool = False
+) -> tuple[mock._Call, ...]:
     cr_dir = config.results_dir / "cosmic_ray"
     config_file = cr_dir / f"{target}.toml"
     db_file = cr_dir / f"{target}.sqlite"
-    return (
+    calls: tuple[mock._Call, ...] = (
         mock.call(config, "cosmic-ray", "init", str(config_file), str(db_file)),
-        mock.call(config, "cosmic-ray", "baseline", str(config_file)),
-        mock.call(config, "cosmic-ray", "exec", str(config_file), str(db_file)),
-        mock.call(config, "cr-report", str(db_file), capture=True),
+        mock.call(
+            config,
+            "cosmic-ray",
+            "baseline",
+            str(config_file),
+            show_output_on_error=False,
+        ),
     )
+    if not skip_exec:
+        calls = (
+            *calls,
+            mock.call(config, "cosmic-ray", "exec", str(config_file), str(db_file)),
+        )
+    return (*calls, mock.call(config, "cr-report", str(db_file), capture=True))
 
 
 def _cr_config(target: Path, test_file: Path | None) -> str:
@@ -136,11 +242,14 @@ def _cr_config(target: Path, test_file: Path | None) -> str:
 
 
 class _OutputCounter:
-    def __init__(self) -> None:
+    def __init__(self, *, fail_baseline: bool = False) -> None:
         self._total_count = 9
         self._successful_count = -1
+        self._fail_baseline = fail_baseline
 
     def __call__(self, _config: Config, *args: str, **_kwargs: object) -> str:
+        if self._fail_baseline and args[0:2] == ("cosmic-ray", "baseline"):
+            raise CommandFailedError(args)
         if args[0] != "cr-report":
             return ""
         self._total_count += 1