From 2c1d36caa4406771db5b1ee2bdcfedbc13d8400a Mon Sep 17 00:00:00 2001 From: Tyson Smith Date: Thu, 4 Jan 2024 17:53:13 -0800 Subject: [PATCH] Add Report.short_signature --- grizzly/common/report.py | 26 +++++++++++++++++++++++ grizzly/common/test_report.py | 39 ++++++++++++++++++++++++++++------ grizzly/reduce/core.py | 7 ++---- grizzly/replay/replay.py | 20 ++++++------------ grizzly/replay/test_replay.py | 13 ------------ grizzly/session.py | 19 +++++------------ grizzly/test_session.py | 40 +++++++++++++++++++++++++++-------- 7 files changed, 104 insertions(+), 60 deletions(-) diff --git a/grizzly/common/report.py b/grizzly/common/report.py index 0e68cae5..dda93a68 100644 --- a/grizzly/common/report.py +++ b/grizzly/common/report.py @@ -42,6 +42,7 @@ class Report: "_crash_info", "_logs", "_signature", + "_short_signature", "_target_binary", "is_hang", "path", @@ -55,6 +56,7 @@ def __init__(self, log_path, target_binary, is_hang=False, size_limit=MAX_LOG_SI self._crash_info = None self._logs = self._select_logs(log_path) assert self._logs is not None + self._short_signature = None self._signature = None self._target_binary = target_binary self.is_hang = is_hang @@ -118,6 +120,9 @@ def crash_hash(self): Returns: str: Hash of the raw signature of the crash. """ + if self.is_hang: + # TODO: we cannot create a unique bucket hash for hangs atm + return "hang" return self.calc_hash(self.crash_signature) @property @@ -411,6 +416,27 @@ def _select_logs(cls, path): result = LogMap(log_aux, log_err, log_out) return result if any(result) else None + @property + def short_signature(self): + """Short signature of the report created by FuzzManager. + + Args: + None + + Returns: + str: Short signature. + """ + if self._short_signature is None: + if self.is_hang: + # TODO: remove once we can create accurate signatures for hangs + self._short_signature = "Potential hang detected" + elif self.crash_signature is None: + # FM crash signature creation failed + self._short_signature = "Signature creation failed" + else: + self._short_signature = self.crash_info.createShortSignature() + return self._short_signature + @staticmethod def tail(in_file, size_limit): """Tail the given file. WARNING: This is destructive! diff --git a/grizzly/common/test_report.py b/grizzly/common/test_report.py index 4234e844..56664148 100644 --- a/grizzly/common/test_report.py +++ b/grizzly/common/test_report.py @@ -6,7 +6,7 @@ from pathlib import Path from FTB.Signatures.CrashInfo import CrashInfo -from pytest import mark, raises +from pytest import mark from .report import Report @@ -62,10 +62,10 @@ def test_report_03(tmp_path): tmp_file = tmp_path / "file.txt" tmp_file.write_bytes(b"blah\ntest\n123\xEF\x00FOO") length = tmp_file.stat().st_size - # no size limit - with raises(AssertionError): - Report.tail(tmp_file, 0) + # don't trim + Report.tail(tmp_file, length + 1) assert tmp_file.stat().st_size == length + # perform trim Report.tail(tmp_file, 3) log_data = tmp_file.read_bytes() assert log_data.startswith(b"[LOG TAILED]\n") @@ -286,16 +286,19 @@ def test_report_12(tmp_path): assert report._crash_info is None assert report.crash_info is not None assert report._crash_info is not None + assert report._crash_info.configuration.product == "bin" # with binary.fuzzmanagerconf - with (tmp_path / "fake_bin.fuzzmanagerconf").open("wb") as conf: + with (tmp_path / "bin.fuzzmanagerconf").open("wb") as conf: conf.write(b"[Main]\n") conf.write(b"platform = x86-64\n") - conf.write(b"product = mozilla-central\n") + conf.write(b"product = grizzly-test\n") conf.write(b"os = linux\n") report = Report(tmp_path, Path("bin")) + report._target_binary = tmp_path / "bin" assert report._crash_info is None assert report.crash_info is not None assert report._crash_info is not None + assert report._crash_info.configuration.product == "grizzly-test" @mark.parametrize( @@ -382,3 +385,27 @@ def test_report_15(tmp_path, data, lines): log = tmp_path / "test-log.txt" log.write_text(data) assert len(Report._load_log(log)) == lines + + +@mark.parametrize( + "hang, has_log, expected", + [ + # process log and create short signature + (False, True, "[@ foo]"), + # no log available to create short signature + (False, False, "Signature creation failed"), + # result is a hang + (True, True, "Potential hang detected"), + ], +) +def test_report_16(mocker, tmp_path, hang, has_log, expected): + """test Report.short_signature""" + mocker.patch("grizzly.common.report.ProgramConfiguration", autospec=True) + (tmp_path / "log_stderr.txt").write_bytes(b"STDERR log") + (tmp_path / "log_stdout.txt").write_bytes(b"STDOUT log") + if has_log: + _create_crash_log(tmp_path / "log_asan_blah.txt") + report = Report(tmp_path, Path("bin"), is_hang=hang) + assert report.short_signature == expected + if hang: + assert report.crash_hash == "hang" diff --git a/grizzly/reduce/core.py b/grizzly/reduce/core.py index 99206fd6..e034c33a 100644 --- a/grizzly/reduce/core.py +++ b/grizzly/reduce/core.py @@ -330,9 +330,7 @@ def run_reliability_analysis(self): first_expected = next( (report for report in results if report.expected), None ) - self._signature_desc = ( - first_expected.report.crash_info.createShortSignature() - ) + self._signature_desc = first_expected.report.short_signature self.report( [result for result in results if not result.expected], testcases, @@ -543,8 +541,7 @@ def run(self, repeat=1, launch_attempts=3, min_results=1, post_launch_delay=0): not self._any_crash and self._signature_desc is None ): - crash = first_expected.report.crash_info - sig = crash.createShortSignature() + sig = first_expected.report.short_signature self._signature_desc = sig self._status.report() strategy.update(success) diff --git a/grizzly/replay/replay.py b/grizzly/replay/replay.py index 8ee89fdc..09ab3130 100644 --- a/grizzly/replay/replay.py +++ b/grizzly/replay/replay.py @@ -409,15 +409,6 @@ def run( report = Report( log_path, self.target.binary, is_hang=run_result.timeout ) - # check signatures - if run_result.timeout: - short_sig = "Potential hang detected" - elif report.crash_signature is not None: - short_sig = report.crash_info.createShortSignature() - else: - # FM crash signature creation failed - short_sig = "Signature creation failed" - # set active signature if ( not runner.startup_failure @@ -427,7 +418,10 @@ def run( ): assert not expect_hang assert self._signature is None - LOG.debug("no signature given, using short sig %r", short_sig) + LOG.debug( + "no signature given, using short sig %r", + report.short_signature, + ) self._signature = report.crash_signature sig_set = True if self._signature is not None: @@ -446,10 +440,10 @@ def run( bucket_hash = sig_hash else: bucket_hash = report.crash_hash - self.status.results.count(bucket_hash, short_sig) + self.status.results.count(bucket_hash, report.short_signature) LOG.info( "Result: %s (%s:%s)", - short_sig, + report.short_signature, report.major[:8], report.minor[:8], ) @@ -463,7 +457,7 @@ def run( else: LOG.info( "Result: Different signature: %s (%s:%s)", - short_sig, + report.short_signature, report.major[:8], report.minor[:8], ) diff --git a/grizzly/replay/test_replay.py b/grizzly/replay/test_replay.py index a1c841e5..3dab34a8 100644 --- a/grizzly/replay/test_replay.py +++ b/grizzly/replay/test_replay.py @@ -299,11 +299,8 @@ def test_replay_09(mocker, server): """test ReplayManager.run() - test signatures - fail to meet minimum""" mocker.patch("grizzly.common.runner.sleep", autospec=True) report_1 = mocker.Mock(spec_set=Report, crash_hash="h1", major="0123", minor="0123") - report_1.crash_info.createShortSignature.return_value = "[@ test1]" report_2 = mocker.Mock(spec_set=Report, crash_hash="h2", major="0123", minor="abcd") - report_2.crash_info.createShortSignature.return_value = "[@ test2]" report_3 = mocker.Mock(spec_set=Report, crash_hash="h2", major="0123", minor="abcd") - report_3.crash_info.createShortSignature.return_value = "[@ test2]" fake_report = mocker.patch("grizzly.replay.replay.Report", autospec=True) fake_report.side_effect = (report_1, report_2, report_3) fake_report.calc_hash.return_value = "bucketHASH" @@ -335,9 +332,7 @@ def test_replay_09(mocker, server): def test_replay_10(mocker, server): """test ReplayManager.run() - test signatures - multiple matches""" report_0 = mocker.Mock(spec_set=Report, crash_hash="h1", major="0123", minor="0123") - report_0.crash_info.createShortSignature.return_value = "[@ test1]" report_1 = mocker.Mock(spec_set=Report, crash_hash="h2", major="0123", minor="abcd") - report_1.crash_info.createShortSignature.return_value = "[@ test2]" fake_report = mocker.patch("grizzly.replay.replay.Report", autospec=True) fake_report.side_effect = (report_0, report_1) fake_report.calc_hash.return_value = "bucketHASH" @@ -367,9 +362,7 @@ def test_replay_10(mocker, server): def test_replay_11(mocker, server): """test ReplayManager.run() - any crash - success""" report_1 = mocker.Mock(spec_set=Report, crash_hash="h1", major="0123", minor="0123") - report_1.crash_info.createShortSignature.return_value = "[@ test1]" report_2 = mocker.Mock(spec_set=Report, crash_hash="h2", major="0123", minor="abcd") - report_2.crash_info.createShortSignature.return_value = "[@ test2]" fake_report = mocker.patch("grizzly.replay.replay.Report", autospec=True) fake_report.side_effect = (report_1, report_2) server.serve_path.return_value = (Served.ALL, {"a.html": "/fake/path"}) @@ -395,9 +388,7 @@ def test_replay_11(mocker, server): def test_replay_12(mocker, server): """test ReplayManager.run() - any crash - fail to meet minimum""" report_1 = mocker.Mock(spec_set=Report, crash_hash="h1", major="0123", minor="0123") - report_1.crash_info.createShortSignature.return_value = "[@ test1]" report_2 = mocker.Mock(spec_set=Report, crash_hash="h2", major="0123", minor="abcd") - report_2.crash_info.createShortSignature.return_value = "[@ test2]" fake_report = mocker.patch("grizzly.replay.replay.Report", autospec=True) fake_report.side_effect = (report_1, report_2) server.serve_path.return_value = (Served.ALL, {"a.html": "/fake/path"}) @@ -446,14 +437,11 @@ def test_replay_14(mocker, server): auto_sig.matches.side_effect = (True, False, True) # original report_1 = mocker.Mock(spec_set=Report, crash_hash="h1", major="012", minor="999") - report_1.crash_info.createShortSignature.return_value = "[@ test1]" report_1.crash_signature = auto_sig # non matching report report_2 = mocker.Mock(spec_set=Report, crash_hash="h2", major="abc", minor="987") - report_2.crash_info.createShortSignature.return_value = "[@ test2]" # matching report report_3 = mocker.Mock(spec_set=Report, crash_hash="h1", major="012", minor="999") - report_3.crash_info.createShortSignature.return_value = "[@ test1]" fake_report = mocker.patch("grizzly.replay.replay.Report", autospec=True) fake_report.side_effect = (report_1, report_2, report_3) fake_report.calc_hash.return_value = "bucket_hash" @@ -481,7 +469,6 @@ def test_replay_14(mocker, server): def test_replay_15(mocker, server): """test ReplayManager.run() - unexpected exception""" report_0 = mocker.Mock(spec_set=Report, crash_hash="h1", major="0123", minor="0123") - report_0.crash_info.createShortSignature.return_value = "[@ test1]" fake_report = mocker.patch("grizzly.replay.replay.Report", autospec=True) fake_report.side_effect = (report_0,) server.serve_path.side_effect = ( diff --git a/grizzly/session.py b/grizzly/session.py index 1465321e..826eaff8 100644 --- a/grizzly/session.py +++ b/grizzly/session.py @@ -245,26 +245,17 @@ def run( if result.status == Result.FOUND: LOG.debug("result detected") report = self.target.create_report(is_hang=result.timeout) - if result.timeout: - # TODO: we cannot create a unique bucket hash for hangs atm - bucket_hash = "hang" - short_sig = "Potential hang detected" - else: - bucket_hash = report.crash_hash - if report.crash_signature is not None: - short_sig = report.crash_info.createShortSignature() - else: - # FM crash signature creation failed - short_sig = "Signature creation failed" - seen, initial = self.status.results.count(bucket_hash, short_sig) + seen, initial = self.status.results.count( + report.crash_hash, report.short_signature + ) LOG.info( "Result: %s (%s:%s) - %d", - short_sig, + report.short_signature, report.major[:8], report.minor[:8], seen, ) - if initial or not self.status.results.is_frequent(bucket_hash): + if initial or not self.status.results.is_frequent(report.crash_hash): # add target info to test cases for test in self.iomanager.tests: test.assets = dict(self.target.asset_mgr.assets) diff --git a/grizzly/test_session.py b/grizzly/test_session.py index 3f9229b3..0f091060 100644 --- a/grizzly/test_session.py +++ b/grizzly/test_session.py @@ -253,9 +253,12 @@ def test_session_05(mocker, harness, report_size): """test Session - handle Target delayed failures""" reporter = mocker.Mock(spec_set=Reporter) report = mocker.Mock( - spec_set=Report, major="major123", minor="minor456", crash_hash="1234" + spec_set=Report, + major="major123", + minor="minor456", + crash_hash="1234", + short_signature="[@ sig]", ) - report.crash_info.createShortSignature.return_value = "[@ sig]" server = mocker.Mock(spec_set=Sapphire, port=0x1337) target = mocker.MagicMock(spec_set=Target, launch_timeout=30) target.monitor.launches = 1 @@ -293,9 +296,12 @@ def test_session_05(mocker, harness, report_size): def test_session_06(mocker, srv_results, target_result, ignored, results): """test Session.run() - initial test case was not served""" report = mocker.Mock( - spec_set=Report, major="major123", minor="minor456", crash_hash="123" + spec_set=Report, + major="major123", + minor="minor456", + crash_hash="123", + short_signature="[@ sig]", ) - report.crash_info.createShortSignature.return_value = "[@ sig]" reporter = mocker.Mock(spec_set=Reporter) server = mocker.Mock(spec_set=Sapphire, port=0x1337) target = mocker.MagicMock(spec_set=Target, closed=True, launch_timeout=10) @@ -344,7 +350,13 @@ def test_session_08(mocker): runner.return_value.run.return_value = result runner.return_value.startup_failure = False adapter = mocker.Mock(spec_set=Adapter, remaining=None) - report = mocker.Mock(spec_set=Report, major="major123", minor="minor456") + report = mocker.Mock( + spec_set=Report, + major="major123", + minor="minor456", + crash_hash="hang", + short_signature="[@ sig]", + ) reporter = mocker.Mock(spec_set=Reporter) server = mocker.Mock(spec_set=Sapphire, port=0x1337) target = mocker.MagicMock(spec_set=Target, environ={}) @@ -375,8 +387,13 @@ def test_session_09(mocker, harness, report_size, relaunch, iters, report_limit) """test Session - limit report submission""" adapter = SimpleAdapter(harness) reporter = mocker.Mock(spec_set=Reporter) - report = mocker.Mock(spec_set=Report, major="abc", minor="def", crash_hash="123") - report.crash_info.createShortSignature.return_value = "[@ sig]" + report = mocker.Mock( + spec_set=Report, + major="abc", + minor="def", + crash_hash="123", + short_signature="[@ sig]", + ) server = mocker.Mock(spec_set=Sapphire, port=0x1337) target = mocker.MagicMock(spec_set=Target, launch_timeout=30) target.monitor.launches = 1 @@ -417,8 +434,13 @@ def test_session_10(mocker, harness, iters, result_limit, results): """test Session - limit results""" adapter = SimpleAdapter(harness) reporter = mocker.Mock(spec_set=Reporter) - report = mocker.Mock(spec_set=Report, major="abc", minor="def", crash_hash="123") - report.crash_info.createShortSignature.return_value = "[@ sig]" + report = mocker.Mock( + spec_set=Report, + major="abc", + minor="def", + crash_hash="123", + short_signature="[@ sig]", + ) server = mocker.Mock(spec_set=Sapphire, port=0x1337) target = mocker.MagicMock(spec_set=Target, launch_timeout=30) target.monitor.launches = 1