diff --git a/grizzly/reduce/core.py b/grizzly/reduce/core.py index 98f4c171..3ba90c07 100644 --- a/grizzly/reduce/core.py +++ b/grizzly/reduce/core.py @@ -269,13 +269,13 @@ def run_reliability_analysis(self): # if `self.testcases` only has 1 entry to begin with, we don't need # `last_test_only` to trim it for use_harness, last_test_only in ( + # don't relaunch between iterations (be quick) (True, True), + # relaunch between iterations regardless of testcase count (be thorough) (True, False), - # only one of the two use_harness=False cases will run: - # input is len(self.testcases)>1 and we will only try the last testcase + # without the harness we only try the last testcase + # relaunch between iterations is implied (False, True), - # input is len(self.testcases)==1 already and there's no need to trim - (False, False), ): if use_harness and (not self._original_use_harness or harness_crashes): # Don't test with harness again if we already found crashes with the @@ -284,15 +284,8 @@ def run_reliability_analysis(self): if not use_harness and harness_crashes >= self.ANALYSIS_ITERATIONS / 2: # Don't test without harness if harness found > 50% crashes continue - if last_test_only and len(self.testcases) == 1: - # Only set `last_test_only` if we initially have more than one testcase - continue - if not use_harness and (not last_test_only and len(self.testcases) > 1): - # Can't run without harness if we have more than one testcase - # (`last_test_only` will run) - continue - if use_harness and (last_test_only or len(self.testcases) == 1): + if use_harness and last_test_only: relaunch = self.ANALYSIS_ITERATIONS else: relaunch = 1 diff --git a/grizzly/reduce/test_reduce.py b/grizzly/reduce/test_reduce.py index def4f672..660f001a 100644 --- a/grizzly/reduce/test_reduce.py +++ b/grizzly/reduce/test_reduce.py @@ -58,9 +58,9 @@ def _fake_save_logs_bar(result_logs): "expected_min_crashes,use_harness,result_harness", [ param( - None, 11, None, + None, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, True, @@ -68,11 +68,21 @@ def _fake_save_logs_bar(result_logs): id="perfect with harness", ), param( - None, + 0, 11, None, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, + True, + True, + id="perfect with harness, relaunch=1", + ), + param( + None, + None, + 11, + ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, + ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, False, False, id="perfect, use_harness=False", @@ -188,7 +198,7 @@ def _fake_save_logs_bar(result_logs): id="1/11 with harness, perfect without", ), param( - None, + 0, 0, 11, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, @@ -197,16 +207,16 @@ def _fake_save_logs_bar(result_logs): False, id="perfect without harness", ), - param(None, 0, 10, 2, 1, True, False, id="10/11 without harness"), - param(None, 0, 9, 2, 1, True, False, id="9/11 without harness"), - param(None, 0, 8, 3, 1, True, False, id="8/11 without harness"), - param(None, 0, 7, 3, 1, True, False, id="7/11 without harness"), - param(None, 0, 6, 4, 1, True, False, id="6/11 without harness"), - param(None, 0, 5, 5, 1, True, False, id="5/11 without harness"), - param(None, 0, 4, 7, 1, True, False, id="4/11 without harness"), - param(None, 0, 3, 10, 1, True, False, id="3/11 without harness"), - param(None, 0, 2, 15, 1, True, False, id="2/11 without harness"), - param(None, 0, 1, 32, 1, True, False, id="1/11 without harness"), + param(0, 0, 10, 2, 1, True, False, id="10/11 without harness"), + param(0, 0, 9, 2, 1, True, False, id="9/11 without harness"), + param(0, 0, 8, 3, 1, True, False, id="8/11 without harness"), + param(0, 0, 7, 3, 1, True, False, id="7/11 without harness"), + param(0, 0, 6, 4, 1, True, False, id="6/11 without harness"), + param(0, 0, 5, 5, 1, True, False, id="5/11 without harness"), + param(0, 0, 4, 7, 1, True, False, id="4/11 without harness"), + param(0, 0, 3, 10, 1, True, False, id="3/11 without harness"), + param(0, 0, 2, 15, 1, True, False, id="2/11 without harness"), + param(0, 0, 1, 32, 1, True, False, id="1/11 without harness"), param( 1, None,