diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 20ef4d42d6..68fa56dd79 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -199,6 +199,25 @@ def parse_target_selectors(selector_str: str) -> set[int]: ################################################################################### +# Exit code returned when the campaign completes successfully but at least one +# mutant escaped the test suite. Distinct from 1 (used by argparse / setup +# errors) so CI scripts can tell the two apart. +EXIT_CODE_UNCAUGHT_MUTANTS = 2 + + +def _exit_for_campaign_results(total_mutants: int, caught_mutants: int) -> None: + """Exit non-zero with a summary if any compiled mutants were uncaught.""" + uncaught = total_mutants - caught_mutants + if uncaught > 0: + logger.info( + red( + f"{uncaught} uncaught mutant(s) out of {total_mutants} that compiled " + f"({caught_mutants} caught)" + ) + ) + sys.exit(EXIT_CODE_UNCAUGHT_MUTANTS) + + def main() -> None: args = parse_args() @@ -213,6 +232,10 @@ def main() -> None: mutators_to_run: list[str] | None = args.mutators_to_run comprehensive_flag: bool | None = args.comprehensive + # accumulators across all files for the campaign-wide summary / exit code + campaign_total_mutants = 0 + campaign_caught_mutants = 0 + logger.info(blue(f"Starting mutation campaign in {args.codebase}")) if paths_to_ignore: @@ -463,6 +486,9 @@ def main() -> None: else: logger.info(magenta("Zero Tweak mutants analyzed\n")) + campaign_total_mutants += sum(total_mutant_counts) + campaign_caught_mutants += sum(caught_mutant_counts) + # Reset mutant counts before moving on to the next file total_mutant_counts[0] = 0 total_mutant_counts[1] = 0 @@ -486,5 +512,7 @@ def main() -> None: blue(f"Finished mutation testing assessment of '{args.codebase}' in {elapsed_string}\n") ) + _exit_for_campaign_results(campaign_total_mutants, campaign_caught_mutants) + # endregion diff --git a/tests/tools/mutator/test_mutator.py b/tests/tools/mutator/test_mutator.py index 6f9af26999..73c26c7275 100644 --- a/tests/tools/mutator/test_mutator.py +++ b/tests/tools/mutator/test_mutator.py @@ -9,7 +9,13 @@ import pytest from slither import Slither -from slither.tools.mutator.__main__ import _get_mutators, main, parse_target_selectors +from slither.tools.mutator.__main__ import ( + EXIT_CODE_UNCAUGHT_MUTANTS, + _exit_for_campaign_results, + _get_mutators, + main, + parse_target_selectors, +) from slither.tools.mutator.utils.testing_generated_mutant import run_test_cmd from slither.tools.mutator.utils.file_handling import get_sol_file_list, backup_source_file from slither.utils.function import get_function_id @@ -255,6 +261,29 @@ def test_should_mutate_function_no_match(solc_binary_path): assert mutator.should_mutate_function(func) is False +def test_exit_for_campaign_results_all_caught_does_not_exit(): + """All compiled mutants caught: helper returns without raising SystemExit.""" + _exit_for_campaign_results(total_mutants=5, caught_mutants=5) + + +def test_exit_for_campaign_results_no_mutants_does_not_exit(): + """No mutants compiled at all (e.g. interface-only run): exit 0.""" + _exit_for_campaign_results(total_mutants=0, caught_mutants=0) + + +def test_exit_for_campaign_results_uncaught_exits_with_code(caplog): + """At least one uncaught mutant: exit with EXIT_CODE_UNCAUGHT_MUTANTS and log a summary.""" + with pytest.raises(SystemExit) as excinfo: + _exit_for_campaign_results(total_mutants=10, caught_mutants=7) + assert excinfo.value.code == EXIT_CODE_UNCAUGHT_MUTANTS + assert excinfo.value.code != 1 + # Surface the actual numbers so CI logs are actionable. + assert any( + "3 uncaught" in record.message and "10 that compiled" in record.message + for record in caplog.records + ) + + def test_should_mutate_function_includes_modifier(solc_binary_path): """Modifier used by target function should be mutated""" solc_path = solc_binary_path("0.8.15")