[reduce] Update analysis

Don't skip running single test case with harness relaunching each iteration. This is more robust and less complex.
MozillaSecurity · Jan 5, 2024 · 2c5c0c3 · 2c5c0c3
1 parent 3c46ee7
commit 2c5c0c3
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 25 deletions.
diff --git a/grizzly/reduce/core.py b/grizzly/reduce/core.py
@@ -269,13 +269,13 @@ def run_reliability_analysis(self):
         # if `self.testcases` only has 1 entry to begin with, we don't need
         # `last_test_only` to trim it
         for use_harness, last_test_only in (
+            # don't relaunch between iterations (be quick)
             (True, True),
+            # relaunch between iterations regardless of testcase count (be thorough)
             (True, False),
-            # only one of the two use_harness=False cases will run:
-            # input is len(self.testcases)>1 and we will only try the last testcase
+            # without the harness we only try the last testcase
+            # relaunch between iterations is implied
             (False, True),
-            # input is len(self.testcases)==1 already and there's no need to trim
-            (False, False),
         ):
             if use_harness and (not self._original_use_harness or harness_crashes):
                 # Don't test with harness again if we already found crashes with the
@@ -284,15 +284,8 @@ def run_reliability_analysis(self):
             if not use_harness and harness_crashes >= self.ANALYSIS_ITERATIONS / 2:
                 # Don't test without harness if harness found > 50% crashes
                 continue
-            if last_test_only and len(self.testcases) == 1:
-                # Only set `last_test_only` if we initially have more than one testcase
-                continue
-            if not use_harness and (not last_test_only and len(self.testcases) > 1):
-                # Can't run without harness if we have more than one testcase
-                # (`last_test_only` will run)
-                continue
 
-            if use_harness and (last_test_only or len(self.testcases) == 1):
+            if use_harness and last_test_only:
                 relaunch = self.ANALYSIS_ITERATIONS
             else:
                 relaunch = 1

diff --git a/grizzly/reduce/test_reduce.py b/grizzly/reduce/test_reduce.py
@@ -58,21 +58,31 @@ def _fake_save_logs_bar(result_logs):
     "expected_min_crashes,use_harness,result_harness",
     [
         param(
-            None,
             11,
             None,
+            None,
             ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES,
             ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES,
             True,
             True,
             id="perfect with harness",
         ),
         param(
-            None,
+            0,
             11,
             None,
             ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES,
             ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES,
+            True,
+            True,
+            id="perfect with harness, relaunch=1",
+        ),
+        param(
+            None,
+            None,
+            11,
+            ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES,
+            ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES,
             False,
             False,
             id="perfect, use_harness=False",
@@ -188,7 +198,7 @@ def _fake_save_logs_bar(result_logs):
             id="1/11 with harness, perfect without",
         ),
         param(
-            None,
+            0,
             0,
             11,
             ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES,
@@ -197,16 +207,16 @@ def _fake_save_logs_bar(result_logs):
             False,
             id="perfect without harness",
         ),
-        param(None, 0, 10, 2, 1, True, False, id="10/11 without harness"),
-        param(None, 0, 9, 2, 1, True, False, id="9/11 without harness"),
-        param(None, 0, 8, 3, 1, True, False, id="8/11 without harness"),
-        param(None, 0, 7, 3, 1, True, False, id="7/11 without harness"),
-        param(None, 0, 6, 4, 1, True, False, id="6/11 without harness"),
-        param(None, 0, 5, 5, 1, True, False, id="5/11 without harness"),
-        param(None, 0, 4, 7, 1, True, False, id="4/11 without harness"),
-        param(None, 0, 3, 10, 1, True, False, id="3/11 without harness"),
-        param(None, 0, 2, 15, 1, True, False, id="2/11 without harness"),
-        param(None, 0, 1, 32, 1, True, False, id="1/11 without harness"),
+        param(0, 0, 10, 2, 1, True, False, id="10/11 without harness"),
+        param(0, 0, 9, 2, 1, True, False, id="9/11 without harness"),
+        param(0, 0, 8, 3, 1, True, False, id="8/11 without harness"),
+        param(0, 0, 7, 3, 1, True, False, id="7/11 without harness"),
+        param(0, 0, 6, 4, 1, True, False, id="6/11 without harness"),
+        param(0, 0, 5, 5, 1, True, False, id="5/11 without harness"),
+        param(0, 0, 4, 7, 1, True, False, id="4/11 without harness"),
+        param(0, 0, 3, 10, 1, True, False, id="3/11 without harness"),
+        param(0, 0, 2, 15, 1, True, False, id="2/11 without harness"),
+        param(0, 0, 1, 32, 1, True, False, id="1/11 without harness"),
         param(
             1,
             None,