Always submit initial results

When a result is detected for the first time it is always submitted.
This can be blocked by the FuzzManager frequent flag. The submission
is forced if the result has only been found once (including parallel
instances). For subsequent submissions we check if the result is
frequent and report accordingly. This takes into account results found
by parallel instances.

grizzly/common/reporter.py

@@ -68,17 +68,18 @@ def _pre_submit(self, report):
         pass
 
     @abstractmethod
-    def _submit_report(self, report, test_cases):
+    def _submit_report(self, report, test_cases, force):
         pass
 
-    def submit(self, test_cases, report):
+    def submit(self, test_cases, report, force=False):
         """Submit report containing results.
 
         Args:
             test_cases (iterable): A collection of testcases, ordered oldest to newest,
                                    the newest being the mostly likely to trigger
                                    the result (crash, assert... etc).
             report (Report): Report to submit.
+            force (bool): Ignore any limits.
 
         Returns:
             *: implementation specific result indicating where the report was created
@@ -99,7 +100,7 @@ def submit(self, test_cases, report):
             else:
                 LOG.info("=== BEGIN REPORT ===\nBrowser hang detected")
             LOG.info("=== END REPORT ===")
-        result = self._submit_report(report, test_cases)
+        result = self._submit_report(report, test_cases, force)
         if report is not None:
             report.cleanup()
         self._post_submit()
@@ -124,7 +125,7 @@ def _pre_submit(self, report):
     def _post_submit(self):
         pass
 
-    def _submit_report(self, report, test_cases):
+    def _submit_report(self, report, test_cases, force):
         # create major bucket directory in working directory if needed
         if self.major_bucket:
             dest = self.report_path / report.major[:16]
@@ -167,13 +168,12 @@ def _post_submit(self):
 class FuzzManagerReporter(Reporter):
     FM_CONFIG = Path.home() / ".fuzzmanagerconf"
 
-    __slots__ = ("_extra_metadata", "force_report", "quality", "tool")
+    __slots__ = ("_extra_metadata", "quality", "tool")
 
     def __init__(self, tool):
         super().__init__()
         assert isinstance(tool, str)
         self._extra_metadata = {}
-        self.force_report = False
         self.quality = Quality.UNREDUCED
         # remove whitespace and use only lowercase
         self.tool = "-".join(tool.lower().split())
@@ -245,10 +245,10 @@ def _ignored(report):
         # ignore Valgrind crashes
         return log_data.startswith("VEX temporary storage exhausted.")
 
-    def _submit_report(self, report, test_cases):
+    def _submit_report(self, report, test_cases, force):
         collector = Collector(tool=self.tool)
 
-        if not self.force_report:
+        if not force:
             if collector.sigCacheDir and Path(collector.sigCacheDir).is_dir():
                 # search for a cached signature match
                 with InterProcessLock(str(grz_tmp() / "fm_sigcache.lock")):

grizzly/common/status.py

@@ -616,9 +616,11 @@ class ResultCounter(SimpleResultCounter):
 
     def __init__(self, pid, db_file, life_time=RESULTS_EXPIRE, report_limit=0):
         super().__init__(pid)
+        assert db_file
         assert report_limit >= 0
         self._db_file = db_file
         self._frequent = set()
+        # use zero to disable report limit
         self._limit = report_limit
         self.last_found = 0
         self._init_db(db_file, pid, life_time)
@@ -667,10 +669,12 @@ def count(self, result_id, desc):
             desc (str): User friendly description.
 
         Returns:
-            int: Current count for given result_id.
+            tuple (int, bool): Local count and initial report (includes
+                               parallel instances) for given result_id.
         """
         super().count(result_id, desc)
         timestamp = time()
+        initial = False
         with closing(connect(self._db_file, timeout=DB_TIMEOUT)) as con:
             cur = con.cursor()
             with con:
@@ -683,6 +687,11 @@ def count(self, result_id, desc):
                     (timestamp, self._count[result_id], self.pid, result_id),
                 )
                 if cur.rowcount < 1:
+                    cur.execute(
+                        """SELECT pid FROM results WHERE result_id = ?;""",
+                        (result_id,),
+                    )
+                    initial = cur.fetchone() is None
                     cur.execute(
                         """INSERT INTO results(
                                 pid,
@@ -691,16 +700,10 @@ def count(self, result_id, desc):
                                 timestamp,
                                 count)
                             VALUES (?, ?, ?, ?, ?);""",
-                        (
-                            self.pid,
-                            result_id,
-                            desc,
-                            timestamp,
-                            self._count[result_id],
-                        ),
+                        (self.pid, result_id, desc, timestamp, self._count[result_id]),
                     )
         self.last_found = timestamp
-        return self._count[result_id]
+        return self._count[result_id], initial
 
     def is_frequent(self, result_id):
         """Scan all results including results from other running instances
@@ -723,16 +726,18 @@ def is_frequent(self, result_id):
         # only check the db for parallel results if
         # - result has been found locally more than once
         # - limit has not been exceeded locally
-        # - a db file is given
-        if self._limit >= total > 1 and self._db_file:
+        if self._limit >= total > 1:
             with closing(connect(self._db_file, timeout=DB_TIMEOUT)) as con:
                 cur = con.cursor()
                 # look up total count from all processes
                 cur.execute(
-                    """SELECT SUM(count) FROM results WHERE result_id = ?;""",
+                    """SELECT COALESCE(SUM(count), 0)
+                        FROM results WHERE result_id = ?;""",
                     (result_id,),
                 )
-                total = cur.fetchone()[0] or 0
+                global_total = cur.fetchone()[0]
+            assert global_total >= total
+            total = global_total
         if total > self._limit:
             self._frequent.add(result_id)
             return True

grizzly/common/test_reporter.py

@@ -47,7 +47,7 @@ def _pre_submit(self, report):
         def _post_submit(self):
             pass
 
-        def _submit_report(self, report, test_cases):
+        def _submit_report(self, report, test_cases, force):
             pass
 
     (tmp_path / "log_stderr.txt").write_bytes(b"log msg")
@@ -208,8 +208,9 @@ def test_fuzzmanager_reporter_02(
     if tests:
         test_cases.append(fake_test)
     reporter = FuzzManagerReporter("fake-tool")
-    reporter.force_report = force
-    reporter.submit(test_cases, Report(log_path, Path("bin"), is_hang=True))
+    reporter.submit(
+        test_cases, Report(log_path, Path("bin"), is_hang=True), force=force
+    )
     assert not log_path.is_dir()
     assert fake_collector.call_args == ({"tool": "fake-tool"},)
     if (frequent and not force) or ignored:

grizzly/common/test_status.py

@@ -576,7 +576,7 @@ def test_report_counter_01(tmp_path, keys, counts, limit):
         assert not counter.is_frequent(report_id)
         # call count() with report_id 'counted' times
         for current in range(1, counted + 1):
-            assert counter.count(report_id, "desc") == current
+            assert counter.count(report_id, "desc") == (current, (current == 1))
         # test get()
         if sum(counts) > 0:
             assert counter.get(report_id) == (report_id, counted, "desc")
@@ -610,31 +610,31 @@ def test_report_counter_02(mocker, tmp_path):
     assert not counter_b.is_frequent("a")
     assert not counter_c.is_frequent("a")
     # local (counter_a, bucket a) count is 1, global (all counters) count is 1
-    assert counter_a.count("a", "desc") == 1
+    assert counter_a.count("a", "desc") == (1, True)
     assert not counter_a.is_frequent("a")
     assert not counter_b.is_frequent("a")
     assert not counter_c.is_frequent("a")
     # local (counter_b, bucket a) count is 1, global (all counters) count is 2
-    assert counter_b.count("a", "desc") == 1
+    assert counter_b.count("a", "desc") == (1, False)
     assert not counter_a.is_frequent("a")
     assert not counter_b.is_frequent("a")
     assert not counter_c.is_frequent("a")
     # local (counter_b, bucket a) count is 2, global (all counters) count is 3
     # locally exceeded
-    assert counter_b.count("a", "desc") == 2
+    assert counter_b.count("a", "desc") == (2, False)
     assert counter_b.is_frequent("a")
     # local (counter_c, bucket a) count is 1, global (all counters) count is 4
-    assert counter_c.count("a", "desc") == 1
+    assert counter_c.count("a", "desc") == (1, False)
     assert not counter_a.is_frequent("a")
     assert counter_b.is_frequent("a")
     assert not counter_c.is_frequent("a")
     # local (counter_a, bucket a) count is 2, global (all counters) count is 5
     # no limit
-    assert counter_a.count("a", "desc") == 2
+    assert counter_a.count("a", "desc") == (2, False)
     assert not counter_a.is_frequent("a")
     # local (counter_c, bucket a) count is 2, global (all counters) count is 6
     # locally not exceeded, globally exceeded
-    assert counter_c.count("a", "desc") == 2
+    assert counter_c.count("a", "desc") == (2, False)
     assert counter_c.is_frequent("a")
     # local (counter_a, bucket x) count is 0, global (all counters) count is 0
     assert not counter_a.is_frequent("x")

grizzly/reduce/core.py

@@ -721,8 +721,6 @@ def report(self, results, testcases, update_status=False):
         for result in results:
             if self._report_to_fuzzmanager:
                 reporter = FuzzManagerReporter(self._report_tool)
-                if result.expected:
-                    reporter.force_report = True
             else:
                 report_dir = "reports" if result.expected else "other_reports"
                 reporter = FilesystemReporter(
@@ -742,7 +740,7 @@ def report(self, results, testcases, update_status=False):
                 if result.served is not None:
                     for clone, served in zip(clones, result.served):
                         clone.purge_optional(served)
-                result = reporter.submit(clones, result.report)
+                result = reporter.submit(clones, result.report, force=result.expected)
                 if result is not None:
                     if isinstance(result, Path):
                         result = str(result)

grizzly/reduce/test_reduce.py

@@ -843,7 +843,8 @@ def replay_run(testcases, _time_limit, **kw):
 
     reporter = mocker.patch("grizzly.reduce.core.FilesystemReporter", autospec=True)
 
-    def submit(test_cases, report):
+    # pylint: disable=unused-argument
+    def submit(test_cases, report, force=False):
         assert test_cases
         assert isinstance(report, Report)
         for test in test_cases:

grizzly/replay/replay.py

@@ -246,8 +246,7 @@ def report_to_fuzzmanager(results, tests, tool=None):
         for result in results:
             # always report expected results
             # avoid reporting unexpected frequent results
-            reporter.force_report = result.expected
-            reporter.submit(tests, result.report)
+            reporter.submit(tests, result.report, force=result.expected)
 
     def run(
         self,

grizzly/session.py

@@ -258,16 +258,16 @@ def run(
                     else:
                         # FM crash signature creation failed
                         short_sig = "Signature creation failed"
-                seen = self.status.results.count(bucket_hash, short_sig)
+                seen, initial = self.status.results.count(bucket_hash, short_sig)
                 LOG.info(
                     "Result: %s (%s:%s) - %d",
                     short_sig,
                     report.major[:8],
                     report.minor[:8],
                     seen,
                 )
-                if not self.status.results.is_frequent(bucket_hash):
-                    self.reporter.submit(self.iomanager.tests, report)
+                if initial or not self.status.results.is_frequent(bucket_hash):
+                    self.reporter.submit(self.iomanager.tests, report, force=initial)
                 else:
                     # we should always submit the first instance of a result
                     assert seen > 1