From e574cd43acd4cb89b3a340724bffe22d13d2a75f Mon Sep 17 00:00:00 2001 From: Tyson Smith Date: Mon, 20 Nov 2023 19:34:58 -0800 Subject: [PATCH] Update TestCase.load() to load in place - Remove TestCase.purge_optional() since it now automatic. - TestCase.load() now only loads a single test case. - Break up test case loading in to smaller more testable parts. - Update Replay and Reduce to support changes - Fix test leaks --- grizzly/adapter/adapter.py | 6 +- grizzly/adapter/no_op_adapter/test_no_op.py | 10 +- grizzly/adapter/test_adapter.py | 10 +- grizzly/common/fuzzmanager.py | 112 ++++-- grizzly/common/runner.py | 17 +- grizzly/common/storage.py | 319 +++++++-------- grizzly/common/test_fuzzmanager.py | 64 ++- grizzly/common/test_runner.py | 92 +++-- grizzly/common/test_storage.py | 420 ++++++++++---------- grizzly/reduce/args.py | 15 + grizzly/reduce/core.py | 89 +++-- grizzly/reduce/strategies/__init__.py | 47 +-- grizzly/reduce/strategies/beautify.py | 11 +- grizzly/reduce/strategies/lithium.py | 73 ++-- grizzly/reduce/strategies/testcases.py | 104 ++--- grizzly/reduce/test_main.py | 40 +- grizzly/reduce/test_reduce.py | 255 ++++++------ grizzly/reduce/test_strategies.py | 287 +++---------- grizzly/reduce/test_strategies_beautify.py | 81 ++-- grizzly/replay/args.py | 42 +- grizzly/replay/crash.py | 2 +- grizzly/replay/replay.py | 80 ++-- grizzly/replay/test_args.py | 30 +- grizzly/replay/test_main.py | 152 +++---- grizzly/replay/test_main_fm.py | 7 +- grizzly/replay/test_replay.py | 297 +++++++------- grizzly/session.py | 3 - grizzly/test_session.py | 39 +- 28 files changed, 1282 insertions(+), 1422 deletions(-) diff --git a/grizzly/adapter/adapter.py b/grizzly/adapter/adapter.py index a3c00446..a50e2339 100644 --- a/grizzly/adapter/adapter.py +++ b/grizzly/adapter/adapter.py @@ -16,11 +16,11 @@ class AdapterError(Exception): class Adapter(metaclass=ABCMeta): - """An Adapter is an interface between Grizzly and a fuzzer. A subclass must + """An Adapter is the interface between Grizzly and a fuzzer. A subclass must be created in order to add support for additional fuzzers. The Adapter is responsible for handling input/output data and executing the fuzzer. It is expected that any processes launched or file created on file system - in the adapter will also be cleaned up in the adapter. + by the adapter will also be cleaned up by the adapter. NOTE: Some methods must not be overloaded doing so will prevent Grizzly from operating correctly. @@ -34,8 +34,6 @@ class Adapter(metaclass=ABCMeta): remaining to process. """ - # Only report test cases with served content. - IGNORE_UNSERVED = True # Maximum iterations between Target relaunches (<1 use default) RELAUNCH = 0 # Maximum execution time per test (used as minimum timeout). The iteration is diff --git a/grizzly/adapter/no_op_adapter/test_no_op.py b/grizzly/adapter/no_op_adapter/test_no_op.py index 6bf6f746..abe8bd70 100644 --- a/grizzly/adapter/no_op_adapter/test_no_op.py +++ b/grizzly/adapter/no_op_adapter/test_no_op.py @@ -11,8 +11,8 @@ def test_no_op_01(): """test a simple Adapter""" adapter = NoOpAdapter("no-op") adapter.setup(None, None) - test = TestCase("a", adapter.name) - assert not test.data_size - assert "a" not in test.contents - adapter.generate(test, None) - assert "a" in test.contents + with TestCase("a", adapter.name) as test: + assert not test.data_size + assert "a" not in test.contents + adapter.generate(test, None) + assert "a" in test.contents diff --git a/grizzly/adapter/test_adapter.py b/grizzly/adapter/test_adapter.py index a9289999..5aef119e 100644 --- a/grizzly/adapter/test_adapter.py +++ b/grizzly/adapter/test_adapter.py @@ -54,24 +54,24 @@ def test_adapter_03(tmp_path): # empty path assert not any(SimpleAdapter.scan_path(str(tmp_path))) # missing path - assert not any(SimpleAdapter.scan_path(str(tmp_path / "none"))) + assert not any(SimpleAdapter.scan_path(tmp_path / "none")) # path to file file1 = tmp_path / "test1.txt" file1.touch() - found = tuple(SimpleAdapter.scan_path(str(file1))) + found = tuple(SimpleAdapter.scan_path(file1)) assert str(file1) in found assert len(found) == 1 # path to directory - assert len(tuple(SimpleAdapter.scan_path(str(tmp_path)))) == 1 + assert len(tuple(SimpleAdapter.scan_path(tmp_path))) == 1 # path to directory (w/ ignored) (tmp_path / ".ignored").touch() nested = tmp_path / "nested" nested.mkdir() file2 = nested / "test2.bin" file2.touch() - assert len(tuple(SimpleAdapter.scan_path(str(tmp_path)))) == 1 + assert len(tuple(SimpleAdapter.scan_path(tmp_path))) == 1 # path to directory (recursive) - found = tuple(SimpleAdapter.scan_path(str(tmp_path), recursive=True)) + found = tuple(SimpleAdapter.scan_path(tmp_path, recursive=True)) assert str(file1) in found assert str(file2) in found assert len(found) == 2 diff --git a/grizzly/common/fuzzmanager.py b/grizzly/common/fuzzmanager.py index 02284c98..761b60dd 100644 --- a/grizzly/common/fuzzmanager.py +++ b/grizzly/common/fuzzmanager.py @@ -5,10 +5,10 @@ import json from contextlib import contextmanager from logging import getLogger -from os import unlink from pathlib import Path from shutil import rmtree -from tempfile import mkdtemp, mkstemp +from tempfile import NamedTemporaryFile, mkdtemp +from zipfile import ZipFile from Collector.Collector import Collector from FTB.ProgramConfiguration import ProgramConfiguration @@ -197,6 +197,16 @@ class CrashEntry: RAW_FIELDS = frozenset({"rawCrashData", "rawStderr", "rawStdout"}) + __slots__ = ( + "_crash_id", + "_coll", + "_contents", + "_data", + "_storage", + "_sig_filename", + "_url", + ) + def __init__(self, crash_id): """Initialize CrashEntry. @@ -206,13 +216,14 @@ def __init__(self, crash_id): assert isinstance(crash_id, int) self._crash_id = crash_id self._coll = Collector() + self._contents = None + self._data = None + self._storage = None + self._sig_filename = None self._url = ( f"{self._coll.serverProtocol}://{self._coll.serverHost}:" f"{self._coll.serverPort}/crashmanager/rest/crashes/{crash_id}/" ) - self._data = None - self._tc_filename = None - self._sig_filename = None @property def crash_id(self): @@ -257,46 +268,79 @@ def cleanup(self): Returns: None """ - if self._tc_filename is not None: - self._tc_filename.unlink() + if self._storage is not None: + rmtree(self._storage, ignore_errors=True) if self._sig_filename is not None: rmtree(self._sig_filename.parent) - def testcase_path(self): - """Download the testcase data from CrashManager. + @staticmethod + def _subset(tests, subset): + """Select a subset of tests directories. Subset values are sanitized to + avoid raising. Arguments: - None + tests list(Path): Directories on disk where testcases exists. + subset list(int): Indices of corresponding directories to select. Returns: - Path: Path on disk where testcase exists_ + list(Path): Directories that have been selected. """ - if self._tc_filename is not None: - return self._tc_filename + assert isinstance(subset, list) + assert tests + count = len(tests) + # deduplicate and limit requested indices to valid range + keep = {max(count + x, 0) if x < 0 else min(x, count - 1) for x in subset} + LOG.debug("using TestCase(s) with index %r", keep) + # build list of items to preserve + return [tests[i] for i in sorted(keep)] + + def testcases(self, subset=None): + """Download the testcase data from CrashManager. + + Arguments: + subset list(int): Indices of corresponding directories to select. - dlurl = self._url + "download/" - response = self._coll.get(dlurl) + Returns: + list(Path): Directories on disk where testcases exists. + """ + if self._contents is None: + assert self._storage is None + response = self._coll.get(f"{self._url}download/") - if "content-disposition" not in response.headers: - raise RuntimeError( - f"Server sent malformed response: {response!r}" - ) # pragma: no cover + if "content-disposition" not in response.headers: + raise RuntimeError( + f"Server sent malformed response: {response!r}" + ) # pragma: no cover + + with NamedTemporaryFile( + dir=grz_tmp("fuzzmanager"), + prefix=f"crash-{self.crash_id}-", + suffix=Path(self.testcase).suffix, + ) as archive: + archive.write(response.content) + archive.seek(0) + # self._storage should be removed when self.cleanup() is called + self._storage = Path( + mkdtemp( + prefix=f"crash-{self.crash_id}-", dir=grz_tmp("fuzzmanager") + ) + ) + with ZipFile(archive) as zip_fp: + zip_fp.extractall(path=self._storage) + # test case directories are named sequentially, a zip with three test + # directories would have: + # - 'foo-2' (oldest) + # - 'foo-1' + # - 'foo-0' (most recent) + # see FuzzManagerReporter for more info + self._contents = sorted( + (x.parent for x in self._storage.rglob("test_info.json")), + reverse=True, + ) - handle, filename = mkstemp( - dir=grz_tmp("fuzzmanager"), - prefix=f"crash-{self.crash_id}-", - suffix=Path(self.testcase).suffix, - ) - try: - with open(handle, "wb") as output: - output.write(response.content) - result = Path(filename) - filename = None - finally: # pragma: no cover - if filename: - unlink(filename) - self._tc_filename = result - return self._tc_filename + if subset and self._contents: + return self._subset(self._contents, subset) + return self._contents def create_signature(self, binary): """Create a CrashManager signature from this crash. diff --git a/grizzly/common/runner.py b/grizzly/common/runner.py index 0c3759d5..ecb90554 100644 --- a/grizzly/common/runner.py +++ b/grizzly/common/runner.py @@ -277,6 +277,9 @@ def run( # overwrite instead of replace 'grz_next_test' for consistency server_map.set_redirect("grz_next_test", "grz_empty", required=True) server_map.set_dynamic_response("grz_empty", lambda _: b"", required=True) + # clear optional contents from test case + # it will be repopulated with served contests + testcase.clear_optional() # serve the test case serve_start = time() server_status, served = self._server.serve_path( @@ -293,13 +296,13 @@ def run( attempted=testcase.entry_point in served, timeout=server_status == Served.TIMEOUT, ) - # add all include files that were served to test case - if server_map.include: - existing = set(testcase.contents) - for url, local_file in served.items(): - if url in existing: - continue - testcase.add_from_file(local_file, file_name=url, copy=True) + # add all files that were served (includes, etc...) to test + existing = set(testcase.required) + for url, local_file in served.items(): + if url in existing: + continue + # use copy here so include files are copied + testcase.add_from_file(local_file, file_name=url, copy=True) # record use of https in testcase testcase.https = self._server.scheme == "https" if result.timeout: diff --git a/grizzly/common/storage.py b/grizzly/common/storage.py index 5be8daaf..15b86542 100644 --- a/grizzly/common/storage.py +++ b/grizzly/common/storage.py @@ -11,8 +11,6 @@ from shutil import copyfile, copytree, move, rmtree from tempfile import NamedTemporaryFile, mkdtemp from time import time -from zipfile import BadZipfile, ZipFile -from zlib import error as zlib_error from .utils import __version__, grz_tmp @@ -52,6 +50,7 @@ class TestCase: "timestamp", "version", "_files", + "_in_place", "_root", ) @@ -59,6 +58,7 @@ def __init__( self, entry_point, adapter_name, + data_path=None, input_fname=None, time_limit=None, timestamp=None, @@ -77,7 +77,12 @@ def __init__( self.timestamp = time() if timestamp is None else timestamp self.version = __version__ self._files = TestFileMap(optional=[], required=[]) - self._root = Path(mkdtemp(prefix="testcase_", dir=grz_tmp("storage"))) + if data_path: + self._root = data_path + self._in_place = True + else: + self._root = Path(mkdtemp(prefix="testcase_", dir=grz_tmp("storage"))) + self._in_place = False def __enter__(self): return self @@ -111,33 +116,37 @@ def add_from_bytes(self, data, file_name, required=False): data_file.unlink(missing_ok=True) def add_from_file(self, src_file, file_name=None, required=False, copy=False): - """Add a file to the TestCase by either copying or moving an existing file. + """Add a file to the TestCase. Copy or move an existing file if needed. Args: src_file (str): Path to existing file to use. file_name (str): Used as file path on disk and URI. Relative to wwwroot. If file_name is not given the name of the src_file will be used. - required (bool): Indicates whether the file must be served. - copy (bool): File will be copied if True otherwise the file will be moved. + required (bool): Indicates whether the file must be served. Typically this + is only used for the entry point. + copy (bool): Copy existing file data. Existing data is moved by default. Returns: None """ src_file = Path(src_file) if file_name is None: - file_name = src_file.name - file_name = self.sanitize_path(file_name) - - test_file = TestFile(file_name, self._root / file_name) - if test_file.file_name in self.contents: - raise TestFileExists(f"{test_file.file_name!r} exists in test") - - test_file.data_file.parent.mkdir(parents=True, exist_ok=True) - if copy: - copyfile(src_file, test_file.data_file) + url_path = self.sanitize_path(src_file.name) else: - move(src_file, test_file.data_file) + url_path = self.sanitize_path(file_name) + if url_path in self.contents: + raise TestFileExists(f"{url_path!r} exists in test") + + test_file = TestFile(url_path, self._root / url_path) + # don't move/copy data is already in place + if src_file.resolve() != test_file.data_file.resolve(): + assert not self._in_place + test_file.data_file.parent.mkdir(parents=True, exist_ok=True) + if copy: + copyfile(src_file, test_file.data_file) + else: + move(src_file, test_file.data_file) # entry_point is always 'required' if required or test_file.file_name == self.entry_point: @@ -154,7 +163,19 @@ def cleanup(self): Returns: None """ - rmtree(self._root, ignore_errors=True) + if not self._in_place: + rmtree(self._root, ignore_errors=True) + + def clear_optional(self): + """Clear optional files. This does not remove data from the file system. + + Args: + None + + Returns: + None + """ + self._files.optional.clear() def clone(self): """Make a copy of the TestCase. @@ -175,11 +196,12 @@ def clone(self): result.assets = dict(self.assets) if result.assets: assert self.assets_path - org_path = self.assets_path try: # copy asset data from test case - result.assets_path = result.root / org_path.relative_to(self.root) - copytree(org_path, result.assets_path) + result.assets_path = result.root / self.assets_path.relative_to( + self.root + ) + copytree(self.assets_path, result.assets_path) except ValueError: # asset data is not part of the test case result.assets_path = self.assets_path @@ -187,7 +209,6 @@ def clone(self): result.env_vars = dict(self.env_vars) result.hang = self.hang result.https = self.https - # copy test data files for entry, required in chain( product(self._files.required, [True]), @@ -211,18 +232,6 @@ def contents(self): for tfile in chain(self._files.required, self._files.optional): yield tfile.file_name - @property - def root(self): - """Location test data is stored on disk. This is intended to be used as wwwroot. - - Args: - None - - Returns: - Path: Directory containing test case files. - """ - return self._root - @property def data_size(self): """Total amount of data used (bytes) by the files in the TestCase. @@ -280,6 +289,26 @@ def dump(self, dst_path, include_details=False): with (dst_path / "test_info.json").open("w") as out_fp: json.dump(info, out_fp, indent=2, sort_keys=True) + @staticmethod + def _find_entry_point(path): + """Locate potential entry point. + + Args: + path (Path): Directory to scan. + + Returns: + Path: Entry point. + """ + entry_point = None + for entry in path.iterdir(): + if entry.suffix.lower() in (".htm", ".html"): + if entry_point is not None: + raise TestCaseLoadFailure("Ambiguous entry point") + entry_point = entry + if entry_point is None: + raise TestCaseLoadFailure("Could not determine entry point") + return entry_point + def get_file(self, path): """Lookup and return the TestFile with the specified file name. @@ -297,13 +326,13 @@ def get_file(self, path): @property def landing_page(self): """TestCase.landing_page is deprecated! - Should be replaced with TestCase.entry_page. + Should be replaced with TestCase.entry_point. Args: None Returns: - str: TestCase.entry_page. + str: TestCase.entry_point. """ LOG.warning( "'TestCase.landing_page' deprecated, use 'TestCase.entry_point' in adapter" @@ -311,95 +340,29 @@ def landing_page(self): return self.entry_point @classmethod - def load(cls, path, adjacent=False): - """Load TestCases from disk. + def load(cls, path, entry_point=None, catalog=False): + """Load a TestCase. Args: path (Path): Path can be: - 1) A directory containing `test_info.json` and data. - 2) A directory with one or more subdirectories of 1. - 3) A zip archive containing testcase data or - subdirectories containing testcase data. - 4) A single file to be used as a test case. - adjacent (bool): Load adjacent files as part of the test case. - This is always the case when loading a directory. - WARNING: This should be used with caution! - - Returns: - list: TestCases successfully loaded from path. - """ - assert isinstance(path, Path) - # unpack archive if needed - if path.name.lower().endswith(".zip"): - unpacked = mkdtemp(prefix="unpack_", dir=grz_tmp("storage")) - try: - with ZipFile(path) as zip_fp: - zip_fp.extractall(path=unpacked) - except (BadZipfile, zlib_error): - rmtree(unpacked, ignore_errors=True) - raise TestCaseLoadFailure("Testcase archive is corrupted") from None - path = Path(unpacked) - else: - unpacked = None - # load testcase data from disk - try: - if path.is_file(): - tests = [cls.load_single(path, adjacent=adjacent)] - elif path.is_dir(): - tests = [] - for tc_path in TestCase.scan_path(path): - tests.append(cls.load_single(tc_path, copy=unpacked is None)) - tests.sort(key=lambda tc: tc.timestamp) - else: - raise TestCaseLoadFailure("Invalid TestCase path") - finally: - if unpacked is not None: - rmtree(unpacked, ignore_errors=True) - return tests - - @classmethod - def load_single(cls, path, adjacent=False, copy=True): - """Load contents of a TestCase from disk. If `path` is a directory it must - contain a valid 'test_info.json' file. - - Args: - path (Path): Path to the directory or file to load. - adjacent (bool): Load adjacent files as part of the TestCase. - This is always true when loading a directory. - WARNING: This should be used with caution! - copy (bool): Files will be copied if True otherwise the they will be moved. + - A single file to be used as a test case. + - A directory containing the test case data. + entry_point (Path): File to use as entry point. + catalog (bool): Scan contents of TestCase.root and track files. + Untracked files will be missed when using clone() or dump(). + Only the entry point will be marked as 'required'. Returns: TestCase: A TestCase. """ assert isinstance(path, Path) - if path.is_dir(): - # load using test_info.json - try: - with (path / "test_info.json").open("r") as in_fp: - info = json.load(in_fp) - except OSError: - raise TestCaseLoadFailure("Missing 'test_info.json'") from None - except ValueError: - raise TestCaseLoadFailure("Invalid 'test_info.json'") from None - if not isinstance(info.get("target"), str): - raise TestCaseLoadFailure("'test_info.json' has invalid 'target' entry") - entry_point = Path(path / info["target"]) - if not entry_point.is_file(): - raise TestCaseLoadFailure( - f"Entry point {info['target']!r} not found in {path}" - ) - # always load all contents of a directory if a 'test_info.json' is loaded - adjacent = True - elif path.is_file(): - entry_point = path - info = {} - else: - raise TestCaseLoadFailure(f"Missing or invalid TestCase '{path}'") - # create testcase and add data + # load test case info + entry_point, info = cls.load_meta(path, entry_point=entry_point) + # create test case test = cls( entry_point.relative_to(entry_point.parent).as_posix(), info.get("adapter", None), + data_path=entry_point.parent, input_fname=info.get("input", None), time_limit=info.get("time_limit", None), timestamp=info.get("timestamp", 0), @@ -414,46 +377,75 @@ def load_single(cls, path, adjacent=False, copy=True): assert isinstance(test.assets, dict) for name, value in test.assets.items(): if not isinstance(name, str) or not isinstance(value, str): - test.cleanup() raise TestCaseLoadFailure("'assets' contains invalid entry") - # copy asset data - src_asset_path = None if test.assets: assets_path = info.get("assets_path", None) - if assets_path and (path / assets_path).is_dir(): - src_asset_path = path / assets_path - test.assets_path = test._root / "_assets_" - copytree(src_asset_path, test.assets_path) - else: + if not assets_path or not (test.root / assets_path).is_dir(): LOG.warning("Could not find assets in test case") test.assets = {} + else: + test.assets_path = test.root / assets_path # sanity check environment variable data assert isinstance(test.env_vars, dict) for name, value in test.env_vars.items(): if not isinstance(name, str) or not isinstance(value, str): - test.cleanup() raise TestCaseLoadFailure("'env' contains invalid entry") - # add entry point - test.add_from_file( - entry_point, file_name=test.entry_point, required=True, copy=copy - ) - # load all adjacent data from directory - if adjacent: - for entry in entry_point.parent.rglob("*"): - if not entry.is_file(): - continue - # ignore asset path - if entry.parent == src_asset_path: - continue - location = entry.relative_to(entry_point.parent).as_posix() - # ignore files that have been previously loaded - if location in (test.entry_point, "test_info.json"): - continue - # NOTE: when loading all files except the entry point are - # marked as `required=False` - test.add_from_file(entry, file_name=location, required=False, copy=copy) + # add contents of directory to test case 'contents' (excluding assets) + # data is not copied/moved because it is already in place + if catalog and path.is_dir(): + # NOTE: only entry point will be marked as 'required' + for entry in test.root.rglob("*"): + if ( + not entry.is_dir() + and test.assets_path not in entry.parents + and entry.name != "test_info.json" + ): + test.add_from_file(entry, entry.relative_to(test.root).as_posix()) + else: + # add entry point + test.add_from_file(entry_point, required=True) return test + @classmethod + def load_meta(cls, path, entry_point=None): + """Process and sanitize TestCase meta data. + + Args: + path (Path): Directory containing test_info.json file. + entry_point (): See TestCase.load(). + + Returns: + tuple(Path, dict): Test case entry point and loaded test info. + """ + assert entry_point is None or isinstance(entry_point, Path) + + # load test case info if available + if path.is_dir(): + try: + info = cls.read_info(path) + except TestCaseLoadFailure as exc: + LOG.info(exc) + info = {} + if entry_point is not None: + info["target"] = entry_point.name + elif info: + entry_point = path / info["target"] + else: + # attempt to determine entry point + entry_point = cls._find_entry_point(path) + if path not in entry_point.parents: + raise TestCaseLoadFailure("Entry point must be in root of given path") + else: + # single file test case + assert entry_point is None + entry_point = path + info = {} + + if not entry_point.exists(): + raise TestCaseLoadFailure(f"Missing or invalid TestCase '{path}'") + + return (entry_point, info) + @property def optional(self): """Get file paths of optional files. @@ -467,25 +459,26 @@ def optional(self): for test in self._files.optional: yield test.file_name - def purge_optional(self, keep): - """Remove optional files that are not in keep. + @staticmethod + def read_info(path): + """Attempt to load test info. Args: - keep (iterable(str)): Files that will not be removed. This can contain - absolute (includes) and relative paths. + path (Path): Directory containing test_info.json. - Returns: - None + Yields: + dict: Test info. """ - to_remove = [] - # iterate over optional files - for idx, opt in enumerate(self._files.optional): - # check entries in 'keep' for a match - if not any(x.endswith(opt.file_name) for x in keep): - to_remove.append(idx) - # purge - for idx in reversed(to_remove): - self._files.optional.pop(idx).data_file.unlink() + try: + with (path / "test_info.json").open("r") as in_fp: + info = json.load(in_fp) + except FileNotFoundError: + info = None + except ValueError: + raise TestCaseLoadFailure("Invalid 'test_info.json'") from None + if info is not None and not isinstance(info.get("target"), str): + raise TestCaseLoadFailure("Invalid 'target' entry in 'test_info.json'") + return info or {} @property def required(self): @@ -500,6 +493,18 @@ def required(self): for test in self._files.required: yield test.file_name + @property + def root(self): + """Location test data is stored on disk. This is intended to be used as wwwroot. + + Args: + None + + Returns: + Path: Directory containing test case files. + """ + return self._root + @staticmethod def sanitize_path(path): """Sanitize given path for use as a URI path. diff --git a/grizzly/common/test_fuzzmanager.py b/grizzly/common/test_fuzzmanager.py index fab6ca74..0c6dacfe 100644 --- a/grizzly/common/test_fuzzmanager.py +++ b/grizzly/common/test_fuzzmanager.py @@ -1,8 +1,10 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +# pylint: disable=protected-access """Tests for interface for getting Crash and Bucket data from CrashManager API""" import json +from zipfile import ZipFile from FTB.ProgramConfiguration import ProgramConfiguration from pytest import mark, raises @@ -160,7 +162,7 @@ def test_crash_2(mocker): assert coll.return_value.get.call_count == 1 -def test_crash_3(mocker): +def test_crash_3(mocker, tmp_path): """crash testcase_path writes and returns testcase zip""" coll = mocker.patch("grizzly.common.fuzzmanager.Collector", autospec=True) coll.return_value.serverProtocol = "http" @@ -168,22 +170,41 @@ def test_crash_3(mocker): coll.return_value.serverHost = "allizom.org" coll.return_value.get.return_value.json.return_value = { "id": 234, - "testcase": "test.bz2", + "testcase": "test.zip", } + # build archive containing test cases + with ZipFile(tmp_path / "test.zip", "w") as zip_fp: + # add files out of order + for i in (1, 3, 2, 0): + test = tmp_path / f"test-{i}" + test.mkdir() + (test / "test_info.json").touch() + zip_fp.write(test / "test_info.json", arcname=f"test-{i}/test_info.json") with CrashEntry(234) as crash: - assert crash.testcase == "test.bz2" # pre-load data dict so I can re-patch get + assert crash.testcase == "test.zip" # pre-load data dict so I can re-patch get coll.return_value.get.return_value = mocker.Mock( - content=b"\x01\x02\x03", + content=(tmp_path / "test.zip").read_bytes(), headers={"content-disposition"}, ) assert coll.return_value.get.call_count == 1 - tc_path = crash.testcase_path() - assert tc_path.is_file() - assert tc_path.suffix == ".bz2" - assert tc_path.read_bytes() == b"\x01\x02\x03" + tests = crash.testcases() + assert len(tests) == 4 + # check order + assert tests[0].name == "test-3" + assert tests[1].name == "test-2" + assert tests[2].name == "test-1" + assert tests[3].name == "test-0" assert coll.return_value.get.call_count == 2 # second call returns same path - assert crash.testcase_path() == tc_path + assert crash.testcases() == tests + # subsets + assert crash.testcases(subset=[0]) == [tests[0]] + # remove second oldest test case (oldest = 0, most recent = n-1) + tests = crash.testcases(subset=[0, 2, 3]) + assert len(tests) == 3 + assert tests[0].name == "test-3" + assert tests[1].name == "test-1" + assert tests[2].name == "test-0" assert coll.return_value.get.call_count == 2 @@ -243,6 +264,31 @@ def test_crash_5(mocker): assert coll.return_value.get.call_count == 1 +def test_crash_6(tmp_path): + """test CrashEntry._subset()""" + # test single entry + paths = [tmp_path / "0"] + assert CrashEntry._subset(paths.copy(), [0]) == paths + assert CrashEntry._subset(paths.copy(), [-1]) == paths + # out of range (should be min/max'ed) + assert CrashEntry._subset(paths.copy(), [1]) == paths + # duplicate index + assert CrashEntry._subset(paths.copy(), [1, 1]) == paths + # test multiple entries select single + paths = [tmp_path / "0", tmp_path / "1", tmp_path / "2"] + assert CrashEntry._subset(paths.copy(), [0]) == [paths[0]] + assert CrashEntry._subset(paths.copy(), [1]) == [paths[1]] + assert CrashEntry._subset(paths.copy(), [2]) == [paths[2]] + # out of range (should be min/max'ed) + assert CrashEntry._subset(paths.copy(), [3]) == [paths[2]] + assert CrashEntry._subset(paths.copy(), [-3]) == [paths[0]] + # test multiple entries select multiple + assert CrashEntry._subset(paths.copy(), [0, 1]) == paths[:2] + assert CrashEntry._subset(paths.copy(), [2, 1]) == paths[1:] + assert CrashEntry._subset(paths.copy(), [0, -1]) == [paths[0], paths[-1]] + assert CrashEntry._subset(paths.copy(), [0, 1, -1]) == paths + + @mark.parametrize( "bucket_id, load_bucket", [ diff --git a/grizzly/common/test_runner.py b/grizzly/common/test_runner.py index ffdc39c6..c93a47c6 100644 --- a/grizzly/common/test_runner.py +++ b/grizzly/common/test_runner.py @@ -29,8 +29,6 @@ def test_runner_01(mocker, coverage, scheme): """test Runner()""" mocker.patch("grizzly.common.runner.time", autospec=True, side_effect=count()) server = mocker.Mock(spec_set=Sapphire, scheme=scheme) - serv_files = ("a.bin", "/another/file.bin") - server.serve_path.return_value = (Served.ALL, serv_files) target = mocker.Mock(spec_set=Target) target.check_result.return_value = Result.NONE runner = Runner(server, target, relaunch=10) @@ -40,7 +38,10 @@ def test_runner_01(mocker, coverage, scheme): assert runner._relaunch == 10 assert runner._tests_run == 0 serv_map = ServerMap() - with TestCase(serv_files[0], "x") as testcase: + with TestCase("a.bin", "x") as testcase: + testcase.add_from_bytes(b"", testcase.entry_point) + serv_files = {"a.bin": testcase.root / "a.bin"} + server.serve_path.return_value = (Served.ALL, serv_files) result = runner.run([], serv_map, testcase, coverage=coverage) assert testcase.https == (scheme == "https") assert runner.initial @@ -48,7 +49,7 @@ def test_runner_01(mocker, coverage, scheme): assert result.attempted assert result.duration == 1 assert result.status == Result.NONE - assert result.served == serv_files + assert result.served == tuple(serv_files) assert not result.timeout assert not result.idle assert not serv_map.dynamic @@ -66,9 +67,12 @@ def test_runner_02(mocker): target = mocker.Mock(spec_set=Target) target.check_result.return_value = Result.NONE serv_files = ("a.bin",) - server.serve_path.return_value = (Served.ALL, serv_files) + server.serve_path.return_value = (Served.ALL, {"a.bin": ""}) testcase = mocker.Mock( - spec_set=TestCase, entry_point=serv_files[0], required=serv_files + spec_set=TestCase, + contents=serv_files, + entry_point=serv_files[0], + required=serv_files, ) # single run/iteration relaunch (not idle exit) target.is_idle.return_value = False @@ -130,9 +134,9 @@ def test_runner_02(mocker): "srv_result, served", [ # no files served - (Served.NONE, []), + (Served.NONE, {}), # entry point not served - (Served.REQUEST, ["harness"]), + (Served.REQUEST, {"harness": ""}), ], ) def test_runner_03(mocker, srv_result, served): @@ -141,9 +145,9 @@ def test_runner_03(mocker, srv_result, served): server.serve_path.return_value = (srv_result, served) target = mocker.Mock(spec_set=Target) target.check_result.return_value = Result.NONE - testcase = mocker.Mock(spec_set=TestCase, entry_point="x", required=["x"]) + test = mocker.Mock(spec_set=TestCase, entry_point="x", required=["x"]) runner = Runner(server, target) - result = runner.run([], ServerMap(), testcase) + result = runner.run([], ServerMap(), test) assert runner.initial assert runner.startup_failure assert result.status == Result.NONE @@ -168,17 +172,22 @@ def test_runner_04(mocker, ignore, status, idle, check_result): """test reporting timeout""" server = mocker.Mock(spec_set=Sapphire) target = mocker.Mock(spec_set=Target) - testcase = mocker.Mock(spec_set=TestCase, entry_point="a.bin", required=["a.bin"]) - serv_files = ("a.bin", "/another/file.bin") + test = mocker.Mock( + spec_set=TestCase, + contents=["a.bin"], + entry_point="a.bin", + required=["a.bin"], + ) + serv_files = {"a.bin": ""} server.serve_path.return_value = (Served.TIMEOUT, serv_files) target.check_result.return_value = Result.FOUND target.handle_hang.return_value = idle target.monitor.is_healthy.return_value = False runner = Runner(server, target, relaunch=1) serv_map = ServerMap() - result = runner.run(ignore, serv_map, testcase) + result = runner.run(ignore, serv_map, test) assert result.status == status - assert result.served == serv_files + assert result.served == tuple(serv_files) assert result.timeout assert result.idle == idle assert "grz_empty" not in serv_map.dynamic @@ -190,11 +199,11 @@ def test_runner_04(mocker, ignore, status, idle, check_result): "served, attempted, target_result, status", [ # FAILURE - (["a.bin"], True, Result.FOUND, Result.FOUND), + ({"a.bin": ""}, True, Result.FOUND, Result.FOUND), # IGNORED - (["a.bin"], True, Result.IGNORED, Result.IGNORED), + ({"a.bin": ""}, True, Result.IGNORED, Result.IGNORED), # failure before serving entry point - (["harness"], False, Result.FOUND, Result.FOUND), + ({"harness": ""}, False, Result.FOUND, Result.FOUND), ], ) def test_runner_05(mocker, served, attempted, target_result, status): @@ -220,14 +229,14 @@ def test_runner_06(mocker): server = mocker.Mock(spec_set=Sapphire) target = mocker.Mock(spec_set=Target) target.check_result.return_value = Result.NONE - serv_files = ("/fake/file", "/another/file.bin") + serv_files = {"a.bin": ""} server.serve_path.return_value = (Served.ALL, serv_files) runner = Runner(server, target, idle_threshold=0.01, idle_delay=0.01, relaunch=10) assert runner._idle is not None result = runner.run( [], ServerMap(), - mocker.Mock(spec_set=TestCase, entry_point=serv_files[0], required=serv_files), + mocker.Mock(spec_set=TestCase, entry_point="a.bin", required=tuple(serv_files)), ) assert result.status == Result.NONE assert result.attempted @@ -328,21 +337,50 @@ def test_runner_10(mocker, tmp_path): smap = ServerMap() smap.set_include("/", str(inc_path1)) smap.set_include("/test", str(inc_path2)) - with TestCase("a.b", "x") as tcase: - tcase.add_from_bytes(b"a", tcase.entry_point, required=True) + with TestCase("a.b", "x") as test: + test.add_from_bytes(b"", test.entry_point) serv_files = { - "a.b": tcase.root / "a.b", + "a.b": test.root / "a.b", "inc_file.bin": inc1, "nested/nested_inc.bin": inc2, "test/inc_file3.txt": inc3, } server.serve_path.return_value = (Served.ALL, serv_files) - result = runner.run([], smap, tcase) + result = runner.run([], smap, test) + assert result.attempted + assert result.status == Result.NONE + assert "inc_file.bin" in test.contents + assert "nested/nested_inc.bin" in test.contents + assert "test/inc_file3.txt" in test.contents + + +def test_runner_11(mocker): + """test Runner.run() - remove unserved and add served test files""" + server = mocker.Mock(spec_set=Sapphire) + target = mocker.Mock(spec_set=Target) + target.check_result.return_value = Result.NONE + runner = Runner(server, target, relaunch=10) + + with TestCase("test.html", "x") as test: + test.add_from_bytes(b"", test.entry_point) + test.add_from_bytes(b"", "other.html") + # add untracked file + (test.root / "extra.js").touch() + assert "extra.html" not in test.contents + assert "other.html" in test.contents + server.serve_path.return_value = ( + Served.ALL, + { + "test.html": test.root / "test.html", + "extra.js": test.root / "extra.js", + }, + ) + result = runner.run([], ServerMap(), test) assert result.attempted assert result.status == Result.NONE - assert "inc_file.bin" in tcase.contents - assert "nested/nested_inc.bin" in tcase.contents - assert "test/inc_file3.txt" in tcase.contents + assert "test.html" in test.contents + assert "extra.js" in test.contents + assert "other.html" not in test.contents @mark.parametrize( @@ -358,7 +396,7 @@ def test_runner_10(mocker, tmp_path): (0, (Served.NONE, None), True), ], ) -def test_runner_11(mocker, delay, srv_result, startup_failure): +def test_runner_12(mocker, delay, srv_result, startup_failure): """test Runner.post_launch()""" srv_timeout = 1 server = mocker.Mock( diff --git a/grizzly/common/test_storage.py b/grizzly/common/test_storage.py index 3803789c..13e19131 100644 --- a/grizzly/common/test_storage.py +++ b/grizzly/common/test_storage.py @@ -3,9 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # pylint: disable=protected-access -from itertools import chain from json import dumps, loads -from zipfile import ZIP_DEFLATED, ZipFile from pytest import mark, raises @@ -31,6 +29,7 @@ def test_testcase_01(tmp_path): assert tcase.root assert not tcase._files.optional assert not tcase._files.required + assert not tcase._in_place assert not any(tcase.contents) assert not any(tcase.optional) assert not any(tcase.required) @@ -38,6 +37,26 @@ def test_testcase_01(tmp_path): assert not any(tmp_path.iterdir()) tcase.dump(tmp_path, include_details=True) assert (tmp_path / "test_info.json").is_file() + tcase.cleanup() + assert not tcase.root.is_dir() + + +def test_testcase_02(): + """test TestCase.add_from_file() - add files with existing, in place data""" + with TestCase("test.html", "adpt") as tcase: + (tcase.root / "test.html").touch() + (tcase.root / "opt.html").touch() + tcase.add_from_file(tcase.root / "test.html", required=True) + assert len(tuple(tcase.contents)) == 1 + assert len(tuple(tcase.required)) == 1 + assert not any(tcase.optional) + tcase.add_from_file(tcase.root / "opt.html", required=False) + assert len(tuple(tcase.contents)) == 2 + assert len(tuple(tcase.required)) == 1 + assert len(tuple(tcase.optional)) == 1 + # add previously added file + with raises(TestFileExists, match="'opt.html' exists in test"): + tcase.add_from_file(tcase.root / "opt.html", required=False) @mark.parametrize( @@ -49,7 +68,7 @@ def test_testcase_01(tmp_path): (False, False), ], ) -def test_testcase_02(tmp_path, copy, required): +def test_testcase_03(tmp_path, copy, required): """test TestCase.add_from_file()""" with TestCase("land_page.html", "adpt", input_fname="in.bin") as tcase: in_file = tmp_path / "file.bin" @@ -80,7 +99,7 @@ def test_testcase_02(tmp_path, copy, required): ("a.bin", "b/c.bin", "b/d.bin"), ], ) -def test_testcase_03(tmp_path, file_paths): +def test_testcase_04(tmp_path, file_paths): """test TestCase.add_from_file()""" with TestCase("land_page.html", "adpt") as tcase: for file_path in file_paths: @@ -91,9 +110,10 @@ def test_testcase_03(tmp_path, file_paths): assert file_path in tcase.contents assert file_path in tcase.required assert file_path not in tcase.optional + assert (tcase.root / file_path).is_file() -def test_testcase_04(): +def test_testcase_05(): """test TestCase.add_from_bytes()""" with TestCase("a.html", "adpt") as tcase: tcase.add_from_bytes(b"foo", "a.html", required=True) @@ -105,37 +125,6 @@ def test_testcase_04(): tcase.add_from_bytes(b"foo", "", required=False) -def test_testcase_05(): - """test TestCase.purge_optional()""" - with TestCase("land_page.html", "test-adapter") as tcase: - # no optional files - tcase.purge_optional(["foo"]) - # setup - tcase.add_from_bytes(b"foo", "testfile1.bin", required=True) - tcase.add_from_bytes(b"foo", "testfile2.bin", required=False) - tcase.add_from_bytes(b"foo", "testfile3.bin", required=False) - tcase.add_from_bytes(b"foo", "not_served.bin", required=False) - assert len(tcase._files.optional) == 3 - # nothing to remove - with required - tcase.purge_optional(chain(["testfile1.bin"], tcase.optional)) - assert len(tcase._files.optional) == 3 - # nothing to remove - use relative path (forced) - tcase.purge_optional(x.file_name for x in tcase._files.optional) - assert len(tcase._files.optional) == 3 - # nothing to remove - use absolute path - tcase.purge_optional(x.data_file.as_posix() for x in tcase._files.optional) - assert len(tcase._files.optional) == 3 - # remove not_served.bin - tcase.purge_optional(["testfile2.bin", "testfile3.bin"]) - assert len(tcase._files.optional) == 2 - assert "testfile2.bin" in tcase.optional - assert "testfile3.bin" in tcase.optional - assert "not_served.bin" not in tcase.optional - # remove remaining optional - tcase.purge_optional(["testfile1.bin"]) - assert not tcase._files.optional - - def test_testcase_06(): """test TestCase.data_size""" with TestCase("land_page.html", "test-adapter") as tcase: @@ -147,155 +136,243 @@ def test_testcase_06(): def test_testcase_07(tmp_path): - """test TestCase.load_single() using a directory - fail cases""" + """test TestCase.read_info()""" # missing test_info.json - with raises(TestCaseLoadFailure, match="Missing 'test_info.json'"): - TestCase.load_single(tmp_path) + assert not TestCase.read_info(tmp_path) # invalid test_info.json - (tmp_path / "test_info.json").write_bytes(b"X") + (tmp_path / "test_info.json").write_text("X") with raises(TestCaseLoadFailure, match="Invalid 'test_info.json'"): - TestCase.load_single(tmp_path) + TestCase.read_info(tmp_path) # test_info.json missing 'target' entry - (tmp_path / "test_info.json").write_bytes(b"{}") + (tmp_path / "test_info.json").write_text("{}") with raises( - TestCaseLoadFailure, match="'test_info.json' has invalid 'target' entry" + TestCaseLoadFailure, match="Invalid 'target' entry in 'test_info.json'" ): - TestCase.load_single(tmp_path) + TestCase.read_info(tmp_path) + # success + (tmp_path / "test_info.json").write_text('{"target": "foo"}') + assert TestCase.read_info(tmp_path) == {"target": "foo"} + + +def test_testcase_08(tmp_path): + """test TestCase._find_entry_point()""" + # empty directory + with raises(TestCaseLoadFailure, match="Could not determine entry point"): + TestCase._find_entry_point(tmp_path) + # missing potential entry point + (tmp_path / "test_info.json").touch() + with raises(TestCaseLoadFailure, match="Could not determine entry point"): + TestCase._find_entry_point(tmp_path) + # success + (tmp_path / "poc.html").touch() + assert TestCase._find_entry_point(tmp_path) == (tmp_path / "poc.html") + # Ambiguous entry point + (tmp_path / "other.html").touch() + with raises(TestCaseLoadFailure, match="Ambiguous entry point"): + TestCase._find_entry_point(tmp_path) + + +def test_testcase_09(tmp_path): + """test TestCase.load_meta()""" + # empty directory + with raises(TestCaseLoadFailure, match="Could not determine entry point"): + TestCase.load_meta(tmp_path) + # missing directory + with raises(TestCaseLoadFailure, match="Missing or invalid TestCase"): + TestCase.load_meta(tmp_path / "missing") + # success (directory) + (tmp_path / "test_01.html").touch() + entry_point, info = TestCase.load_meta(tmp_path) + assert entry_point == tmp_path / "test_01.html" + assert not info + # success (file) + entry_point, info = TestCase.load_meta(tmp_path / "test_01.html") + assert entry_point == tmp_path / "test_01.html" + assert not info + # success (test_info.json) + (tmp_path / "test_info.json").write_text('{"target": "test_01.html"}') + (tmp_path / "other.html").touch() + entry_point, info = TestCase.load_meta(tmp_path) + assert entry_point == (tmp_path / "test_01.html") + assert info.get("target") == "test_01.html" + # success (test_info.json) override entry point + entry_point, info = TestCase.load_meta( + tmp_path, entry_point=(tmp_path / "other.html") + ) + assert entry_point == tmp_path / "other.html" + assert info.get("target") == "other.html" + # invalid test_info.json (will fallback to searching for test) + (tmp_path / "other.html").unlink() + (tmp_path / "test_info.json").write_text("{}") + entry_point, info = TestCase.load_meta(tmp_path) + assert entry_point == (tmp_path / "test_01.html") + assert not info + + +def test_testcase_10(tmp_path): + """test TestCase.load()""" + data = tmp_path / "test-data" + data.mkdir() + # empty directory + with raises(TestCaseLoadFailure, match="Could not determine entry point"): + TestCase.load(tmp_path) + # missing directory + with raises(TestCaseLoadFailure, match="Missing or invalid TestCase"): + TestCase.load(tmp_path / "missing") + # directory with test case + (data / "poc.html").touch() + loaded = TestCase.load(data) + assert loaded._in_place + assert loaded.entry_point == "poc.html" + assert loaded.root == data + # directory with test case invalid entry point + (tmp_path / "external.html").touch() + with raises(TestCaseLoadFailure, match="Entry point must be in root of given path"): + TestCase.load(data, entry_point=tmp_path / "external.html") + # single file directory + loaded = TestCase.load(data / "poc.html") + assert loaded._in_place + assert loaded.entry_point == "poc.html" + assert loaded.root == data + + +def test_testcase_11(tmp_path): + """test TestCase.load() existing test case with simple test_info.json""" + # build a test case + src = tmp_path / "src" + with TestCase("test.html", "test-adapter") as test: + test.add_from_bytes(b"", test.entry_point, required=True) + test.dump(src, include_details=True) + # successful load + loaded = TestCase.load(src) + assert loaded._in_place + assert loaded.entry_point == "test.html" + assert loaded.root.samefile(src) + assert len(tuple(loaded.required)) == 1 + assert not any(loaded.optional) + + +@mark.parametrize("catalog", [False, True]) +def test_testcase_12(tmp_path, catalog): + """test TestCase.load() existing test case with test_info.json""" # build a test case + asset_file = tmp_path / "asset.txt" + asset_file.touch() + src = tmp_path / "src" + with AssetManager(base_path=tmp_path) as asset_mgr: + asset_mgr.add("example", asset_file) + with TestCase("test.html", "test-adapter") as test: + test.add_from_bytes(b"", test.entry_point, required=True) + test.add_from_bytes(b"", "optional.bin", required=False) + test.add_from_bytes(b"", file_name="nested/a.html", required=False) + test.assets = dict(asset_mgr.assets) + test.assets_path = asset_mgr.path + test.env_vars["TEST_ENV_VAR"] = "100" + test.dump(src, include_details=True) + # successful load + loaded = TestCase.load(src, catalog=catalog) + assert loaded.entry_point == "test.html" + assert loaded.root.samefile(src) + assert (loaded.root / "optional.bin").is_file() + assert (loaded.root / "nested" / "a.html").is_file() + if catalog: + assert "optional.bin" in loaded.optional + assert "nested/a.html" in loaded.optional + assert "_assets_/asset.txt" not in loaded.optional + assert "test_info.json" not in loaded.optional + else: + assert not any(loaded.optional) + assert loaded.assets == {"example": "asset.txt"} + assert (loaded.root / loaded.assets_path / "asset.txt").is_file() + assert loaded.env_vars.get("TEST_ENV_VAR") == "100" + assert len(tuple(loaded.required)) == 1 + # this should do nothing + loaded.cleanup() + assert loaded.root.is_dir() + + +def test_testcase_13(tmp_path): + """test TestCase.load() test_info.json error cases""" + # bad 'assets' entry in test_info.json src_dir = tmp_path / "src" src_dir.mkdir() - entry_point = src_dir / "target.bin" - entry_point.touch() - with TestCase("target.bin", "test-adapter") as src: - src.add_from_file(entry_point) - src.dump(src_dir, include_details=True) - # bad 'target' entry in test_info.json - entry_point.unlink() - with raises(TestCaseLoadFailure, match="Entry point 'target.bin' not found in"): - TestCase.load_single(src_dir) - # bad 'assets' entry in test_info.json + entry_point = src_dir / "target.html" entry_point.touch() - with TestCase("target.bin", "test-adapter") as src: + with TestCase("target.html", "test-adapter") as src: src.dump(src_dir, include_details=True) test_info = loads((src_dir / "test_info.json").read_text()) test_info["assets"] = {"bad": 1} (src_dir / "test_info.json").write_text(dumps(test_info)) with raises(TestCaseLoadFailure, match="'assets' contains invalid entry"): - TestCase.load_single(src_dir) + TestCase.load(src_dir) # bad 'env' entry in test_info.json - with TestCase("target.bin", "test-adapter") as src: + with TestCase("target.html", "test-adapter") as src: src.dump(src_dir, include_details=True) test_info = loads((src_dir / "test_info.json").read_text()) test_info["env"] = {"bad": 1} (src_dir / "test_info.json").write_text(dumps(test_info)) with raises(TestCaseLoadFailure, match="'env' contains invalid entry"): - TestCase.load_single(src_dir) + TestCase.load(src_dir) # missing asset data test_info["env"].clear() test_info["assets"] = {"a": "a"} test_info["assets_path"] = "missing" (src_dir / "test_info.json").write_text(dumps(test_info)) - with TestCase.load_single(src_dir) as loaded: + with TestCase.load(src_dir) as loaded: assert not loaded.assets - assert not loaded.assets_path + assert loaded.assets_path is None -def test_testcase_08(tmp_path): - """test TestCase.load_single() using a directory""" - # build a valid test case - src_dir = tmp_path / "src" - src_dir.mkdir() - entry_point = src_dir / "target.bin" - entry_point.touch() - asset_file = src_dir / "asset.bin" +def test_testcase_14(tmp_path): + """test TestCase.load() and TestCase.dump() with assets""" + # build a test case + asset_file = tmp_path / "asset.txt" asset_file.touch() - (src_dir / "optional.bin").touch() - (src_dir / "x.bin").touch() - nested = src_dir / "nested" - nested.mkdir() - # overlap file name in different directories - (nested / "x.bin").touch() - (tmp_path / "src" / "nested" / "empty").mkdir() - dst_dir = tmp_path / "dst" - # build test case - with AssetManager(base_path=tmp_path) as asset_mgr: + src = tmp_path / "src" + with AssetManager() as asset_mgr: asset_mgr.add("example", asset_file) - with TestCase("target.bin", "test-adapter") as src: - src.env_vars["TEST_ENV_VAR"] = "100" - src.add_from_file(entry_point) - src.add_from_file(src_dir / "optional.bin", required=False) - src.add_from_file(src_dir / "x.bin", required=False) - src.add_from_file( - nested / "x.bin", - file_name=str((nested / "x.bin").relative_to(src_dir)), - required=False, - ) - src.assets = dict(asset_mgr.assets) - src.assets_path = asset_mgr.path - src.dump(dst_dir, include_details=True) - # test loading test case from test_info.json - with TestCase.load_single(dst_dir) as dst: - assert "example" in dst.assets - assert dst.assets_path is not None - assert (dst.assets_path / "asset.bin").is_file() - assert "_assets_/asset.bin" not in (x.file_name for x in dst._files.optional) - assert dst.entry_point == "target.bin" - assert "target.bin" in (x.file_name for x in dst._files.required) - assert "optional.bin" in (x.file_name for x in dst._files.optional) - assert "x.bin" in (x.file_name for x in dst._files.optional) - assert "nested/x.bin" in (x.file_name for x in dst._files.optional) - assert dst.env_vars["TEST_ENV_VAR"] == "100" - assert dst.timestamp > 0 - - -def test_testcase_09(tmp_path): - """test TestCase.load_single() using a file""" - # invalid entry_point specified - with raises(TestCaseLoadFailure, match="Missing or invalid TestCase"): - TestCase.load_single(tmp_path / "missing_file", adjacent=False) - # valid test case - src_dir = tmp_path / "src" - src_dir.mkdir() - entry_point = src_dir / "target.bin" - entry_point.touch() - (src_dir / "optional.bin").touch() - # load single file test case - with TestCase.load_single(entry_point, adjacent=False) as tcase: - assert not tcase.assets - assert not tcase.env_vars - assert tcase.entry_point == "target.bin" - assert "target.bin" in (x.file_name for x in tcase._files.required) - assert "optional.bin" not in (x.file_name for x in tcase._files.optional) - assert tcase.timestamp == 0 - # load full test case - with TestCase.load_single(entry_point, adjacent=True) as tcase: - assert tcase.entry_point == "target.bin" - assert "target.bin" in (x.file_name for x in tcase._files.required) - assert "optional.bin" in (x.file_name for x in tcase._files.optional) + with TestCase("test.html", "test-adapter") as test: + test.add_from_bytes(b"", test.entry_point, required=True) + test.assets = dict(asset_mgr.assets) + test.assets_path = asset_mgr.path + test.dump(src, include_details=True) + # load + loaded = TestCase.load(src) + assert loaded.assets + assert loaded.assets_path + with AssetManager.load(loaded.assets, loaded.assets_path) as asset_mgr: + # dump the AssetManager's assets (simulate how this works in replay) + loaded.assets = dict(asset_mgr.assets) + loaded.assets_path = asset_mgr.path + # dump loaded test case + dst = tmp_path / "dst" + loaded.dump(dst, include_details=True) + assert (dst / "test.html").is_file() + assert (dst / "_assets_" / "asset.txt").is_file() -def test_testcase_10(tmp_path): +def test_testcase_15(tmp_path): """test TestCase - dump, load and compare""" asset_path = tmp_path / "assets" asset_path.mkdir() asset = asset_path / "asset_file.txt" asset.touch() working = tmp_path / "working" - working.mkdir() with TestCase("a.html", "adpt") as org: # set non default values org.duration = 1.23 org.env_vars = {"en1": "1", "en2": "2"} - org.https = True - org.hang = True + org.https = not org.https + org.hang = not org.hang org.input_fname = "infile" - org.time_limit = 10 + org.time_limit = 456 org.add_from_bytes(b"a", "a.html") org.assets = {"sample": asset.name} org.assets_path = asset_path + org.version = "1.2.3" org.dump(working, include_details=True) assert (working / "_assets_" / asset.name).is_file() - with TestCase.load_single(working, adjacent=False) as loaded: + with TestCase.load(working) as loaded: for prop in TestCase.__slots__: if not prop.startswith("_") and prop != "assets_path": assert getattr(loaded, prop) == getattr(org, prop) @@ -304,77 +381,6 @@ def test_testcase_10(tmp_path): assert loaded.assets["sample"] == asset.name -def test_testcase_11(tmp_path): - """test TestCase.load() - missing file and empty directory""" - # missing file - with raises(TestCaseLoadFailure, match="Invalid TestCase path"): - TestCase.load(tmp_path / "missing") - # empty path - assert not TestCase.load(tmp_path, adjacent=True) - - -def test_testcase_12(tmp_path): - """test TestCase.load() - single file""" - tfile = tmp_path / "testcase.html" - tfile.touch() - assert len(TestCase.load(tfile, adjacent=False)) == 1 - - -def test_testcase_13(tmp_path): - """test TestCase.load() - single directory""" - with TestCase("target.bin", "test-adapter") as src: - src.add_from_bytes(b"test", "target.bin") - src.dump(tmp_path, include_details=True) - assert len(TestCase.load(tmp_path)) == 1 - - -def test_testcase_14(tmp_path): - """test TestCase.load() - multiple directories (with assets)""" - nested = tmp_path / "nested" - nested.mkdir() - asset_file = tmp_path / "example_asset" - asset_file.touch() - with AssetManager(base_path=tmp_path) as asset_mgr: - asset_mgr.add("example", asset_file) - with TestCase("target.bin", "test-adapter") as src: - src.assets = asset_mgr.assets - src.assets_path = asset_mgr.path - src.add_from_bytes(b"test", "target.bin") - src.dump(nested / "test-1", include_details=True) - src.dump(nested / "test-2", include_details=True) - src.dump(nested / "test-3", include_details=True) - testcases = TestCase.load(nested) - assert len(testcases) == 3 - assert "example" in testcases[0].assets - # try loading testcases that are nested too deep - assert not TestCase.load(tmp_path) - - -def test_testcase_15(tmp_path): - """test TestCase.load() - archive""" - archive = tmp_path / "testcase.zip" - # bad archive - archive.write_bytes(b"x") - with raises(TestCaseLoadFailure, match="Testcase archive is corrupted"): - TestCase.load(archive) - # build archive containing multiple testcases - with TestCase("target.bin", "test-adapter") as src: - src.add_from_bytes(b"test", "target.bin") - src.dump(tmp_path / "test-0", include_details=True) - src.dump(tmp_path / "test-1", include_details=True) - src.dump(tmp_path / "test-2", include_details=True) - (tmp_path / "log_dummy.txt").touch() - (tmp_path / "not_a_tc").mkdir() - (tmp_path / "not_a_tc" / "file.txt").touch() - with ZipFile(archive, mode="w", compression=ZIP_DEFLATED) as zfp: - for entry in tmp_path.rglob("*"): - if entry.is_file(): - zfp.write(str(entry), arcname=str(entry.relative_to(tmp_path))) - testcases = TestCase.load(archive) - assert len(testcases) == 3 - assert all("target.bin" in x.contents for x in testcases) - - def test_testcase_16(tmp_path): """test TestCase.scan_path()""" # empty path diff --git a/grizzly/reduce/args.py b/grizzly/reduce/args.py index 7c83aa14..d2fb32c5 100644 --- a/grizzly/reduce/args.py +++ b/grizzly/reduce/args.py @@ -111,6 +111,21 @@ def __init__(self): "(default: %(default)s).", ) + self.parser.add_argument( + "--test-index", + type=int, + nargs="+", + help="Select a testcase to run when multiple testcases are loaded. " + "Testscases are ordered oldest to newest. " + "0 == oldest, n-1 == most recent (default: run all testcases)", + ) + + def sanity_check(self, args): + super().sanity_check(args) + + if args.no_harness and not args.test_index: + self.parser.error("'--no-harness' requires '--test-index'") + class ReduceFuzzManagerIDQualityArgs(ReduceFuzzManagerIDArgs): def __init__(self): diff --git a/grizzly/reduce/core.py b/grizzly/reduce/core.py index 1dd3a5ff..f19534a2 100644 --- a/grizzly/reduce/core.py +++ b/grizzly/reduce/core.py @@ -157,6 +157,24 @@ def __init__( self._use_analysis = use_analysis self._use_harness = use_harness + def __enter__(self): + return self + + def __exit__(self, *exc): + self.cleanup() + + def cleanup(self): + """Remove temporary files from disk. + + Args: + None + + Returns: + None + """ + for test in self.testcases: + test.cleanup() + def update_timeout(self, results): """Tune idle/server timeout values based on actual duration of expected results. @@ -543,10 +561,7 @@ def run(self, repeat=1, launch_attempts=3, min_results=1, post_launch_delay=0): sig = crash.createShortSignature() self._signature_desc = sig self._status.report() - served = None - if success and not self._any_crash: - served = first_expected.served - strategy.update(success, served=served) + strategy.update(success) if strategy.name == "check" and not success: raise NotReproducible("Not reproducible at 'check'") any_success = any_success or success @@ -571,7 +586,8 @@ def run(self, repeat=1, launch_attempts=3, min_results=1, post_launch_delay=0): test.env_vars = ( self.target.filtered_environ() ) - self.testcases = reduction + # clone results from strategy local copy + self.testcases = [x.clone() for x in reduction] keep_reduction = True # cleanup old best results for result in best_results: @@ -651,6 +667,8 @@ def run(self, repeat=1, launch_attempts=3, min_results=1, post_launch_delay=0): LOG.info("Best results reported (periodic)") finally: + # TODO: TS: I'm not sure this is required anymore + # reduction should only contain strategy local copies if not keep_reduction: for testcase in reduction: testcase.cleanup() @@ -738,21 +756,11 @@ def report(self, results, testcases, update_status=False): ) if self._report_to_fuzzmanager: status.add_to_reporter(reporter, expected=result.expected) - # clone the tests so we can safely call purge_optional here for each report - # (report.served may be different for non-expected or any-crash results) - clones = [test.clone() for test in testcases] - try: - if result.served is not None: - for clone, served in zip(clones, result.served): - clone.purge_optional(served) - result = reporter.submit(clones, result.report, force=result.expected) - if result is not None: - if isinstance(result, Path): - result = str(result) - new_reports.append(result) - finally: - for clone in clones: - clone.cleanup() + result = reporter.submit(testcases, result.report, force=result.expected) + if result is not None: + if isinstance(result, Path): + result = str(result) + new_reports.append(result) # only write new reports if not empty, otherwise previous reports may be # overwritten with an empty list if later reports are ignored if update_status and new_reports: @@ -791,36 +799,29 @@ def main(cls, args): signature_desc = None target = None testcases = [] - try: - if args.sig: - signature = CrashSignature.fromFile(args.sig) - meta = args.sig.with_suffix(".metadata") - if meta.is_file(): - meta = json.loads(meta.read_text()) - signature_desc = meta["shortDescription"] + try: try: testcases, asset_mgr, env_vars = ReplayManager.load_testcases( - args.input, subset=args.test_index + args.input, catalog=True ) except TestCaseLoadFailure as exc: LOG.error("Error: %s", str(exc)) return Exit.ERROR + if args.sig: + signature = CrashSignature.fromFile(args.sig) + meta = args.sig.with_suffix(".metadata") + if meta.is_file(): + meta = json.loads(meta.read_text()) + signature_desc = meta["shortDescription"] + if not args.tool and testcases[0].adapter_name: args.tool = f"grizzly-{testcases[0].adapter_name}" LOG.warning("Setting default --tool=%s from testcase", args.tool) expect_hang = ReplayManager.expect_hang(args.ignore, signature, testcases) - if args.no_harness: - if len(testcases) > 1: - LOG.error( - "Error: '--no-harness' cannot be used with multiple " - "testcases. Perhaps '--test-index' can help." - ) - return Exit.ARGS - # check test time limit and timeout # TODO: add support for test time limit, use timeout in both cases for now _, timeout = time_limits(args.timeout, args.timeout, tests=testcases) @@ -861,7 +862,7 @@ def main(cls, args): # launch HTTP server used to serve test cases with Sapphire(auto_close=1, timeout=timeout, certs=certs) as server: target.reverse(server.port, server.port) - mgr = ReduceManager( + with ReduceManager( args.ignore, server, target, @@ -882,13 +883,13 @@ def main(cls, args): tool=args.tool, use_analysis=not args.no_analysis, use_harness=not args.no_harness, - ) - return_code = mgr.run( - repeat=args.repeat, - launch_attempts=args.launch_attempts, - min_results=args.min_crashes, - post_launch_delay=args.post_launch_delay, - ) + ) as mgr: + return_code = mgr.run( + repeat=args.repeat, + launch_attempts=args.launch_attempts, + min_results=args.min_crashes, + post_launch_delay=args.post_launch_delay, + ) return return_code except ConfigError as exc: diff --git a/grizzly/reduce/strategies/__init__.py b/grizzly/reduce/strategies/__init__.py index dafaaf2a..8bfb7c2d 100644 --- a/grizzly/reduce/strategies/__init__.py +++ b/grizzly/reduce/strategies/__init__.py @@ -161,8 +161,8 @@ def dump_testcases(self, testcases, recreate_tcroot=False): self._testcase_root.mkdir() for idx, testcase in enumerate(testcases): LOG.debug("Extracting testcase %d/%d", idx + 1, len(testcases)) - testpath = self._testcase_root / f"{idx:03d}" - testcase.dump(testpath, include_details=True) + # NOTE: naming determines load order + testcase.dump(self._testcase_root / f"{idx:03d}", include_details=True) @classmethod def sanity_check_cls_attrs(cls): @@ -192,13 +192,11 @@ def __iter__(self): """ @abstractmethod - def update(self, success, served=None): + def update(self, success): """Inform the strategy whether or not the last reduction yielded was good. Arguments: success (bool): Whether or not the last reduction was acceptable. - served (list(list(str))): The list of served files for each testcase in the - last reduction. Returns: None @@ -231,44 +229,7 @@ def cleanup(self): Returns: None """ - rmtree(str(self._testcase_root)) - - def purge_unserved(self, testcases, served): - """Given the testcase list yielded and list of what was served, purge - everything in testcase root to hold only what was served. - - Arguments: - testcases (list(grizzly.common.storage.TestCase): testcases last replayed - served (list(list(str))): list of files served for each testcase. - - Returns: - bool: True if anything was purged - """ - LOG.debug("purging from %d testcases", len(testcases)) - anything_purged = False - while len(served) < len(testcases): - LOG.debug( - "not all %d testcases served (%d served), popping one", - len(testcases), - len(served), - ) - testcases.pop().cleanup() - anything_purged = True - remove_testcases = [] - for idx, (testcase, tc_served) in enumerate(zip(testcases, served)): - LOG.debug("testcase %d served %r", idx, tc_served) - if testcase.entry_point not in tc_served: - LOG.debug("entry point not served (%r)", testcase.entry_point) - remove_testcases.append(idx) - anything_purged = True - else: - size_before = testcase.data_size - testcase.purge_optional(tc_served) - anything_purged = anything_purged or testcase.data_size != size_before - for idx in reversed(remove_testcases): - testcases.pop(idx).cleanup() - self.dump_testcases(testcases, recreate_tcroot=True) - return anything_purged + rmtree(self._testcase_root) STRATEGIES = _load_strategies() diff --git a/grizzly/reduce/strategies/beautify.py b/grizzly/reduce/strategies/beautify.py index af60af11..1ef592f0 100644 --- a/grizzly/reduce/strategies/beautify.py +++ b/grizzly/reduce/strategies/beautify.py @@ -115,19 +115,15 @@ def sanity_check_cls_attrs(cls): assert isinstance(cls.native_extension, str) assert isinstance(cls.tag_name, str) - def update(self, success, served=None): + def update(self, success): """Inform the strategy whether or not the last beautification yielded was good. Arguments: success (bool): Whether or not the last beautification was acceptable. - served (list(list(str))): The list of served files for each testcase in the - last beautification. Returns: None """ - # beautify does nothing with served. it's unlikely a beautify operation alone - # would render a file unserved. assert self._current_feedback is None self._current_feedback = success @@ -277,7 +273,10 @@ def __iter__(self): lith_tc.dump(file) continue - yield TestCase.load(self._testcase_root, True) + testcases = [] + for test in sorted(self._testcase_root.iterdir()): + testcases.append(TestCase.load(test)) + yield testcases assert self._current_feedback is not None, "No feedback for last iteration" if self._current_feedback: diff --git a/grizzly/reduce/strategies/lithium.py b/grizzly/reduce/strategies/lithium.py index e1fdc277..79e17387 100644 --- a/grizzly/reduce/strategies/lithium.py +++ b/grizzly/reduce/strategies/lithium.py @@ -43,13 +43,15 @@ def __init__(self, testcases): testcases. """ super().__init__(testcases) + self._current_feedback = None self._current_reducer = None self._files_to_reduce = [] - self.rescan_files_to_reduce() - self._current_feedback = None - self._current_served = None + local_tests = [] + for test in sorted(self._testcase_root.iterdir()): + local_tests.append(TestCase.load(test, catalog=True)) + self.rescan_files_to_reduce(local_tests) - def rescan_files_to_reduce(self): + def rescan_files_to_reduce(self, testcases): """Repopulate the private `files_to_reduce` attribute by scanning the testcase root. @@ -57,8 +59,8 @@ def rescan_files_to_reduce(self): None """ self._files_to_reduce.clear() - for path in self._testcase_root.glob("**/*"): - if path.is_file() and path.name not in {"test_info.json", "prefs.js"}: + for test in testcases: + for path in (test.root / x for x in test.contents): if _contains_dd(path): self._files_to_reduce.append(path) @@ -76,13 +78,11 @@ def sanity_check_cls_attrs(cls): assert issubclass(cls.strategy_cls, LithStrategy) assert issubclass(cls.testcase_cls, LithTestcase) - def update(self, success, served=None): + def update(self, success): """Inform the strategy whether or not the last reduction yielded was good. Arguments: success (bool): Whether or not the last reduction was acceptable. - served (list(list(str))): The list of served files for each testcase in the - last reduction. Returns: None @@ -90,7 +90,6 @@ def update(self, success, served=None): if self._current_reducer is not None: self._current_reducer.feedback(success) self._current_feedback = success - self._current_served = served def __iter__(self): """Iterate over potential reductions of testcases according to this strategy. @@ -107,11 +106,13 @@ def __iter__(self): file_no = 0 reduce_queue = self._files_to_reduce.copy() reduce_queue.sort() # not necessary, but helps make tests more predictable - # indicates that self._testcase_root contains changes that haven't been yielded - # (if iteration ends, changes would be lost) - testcase_root_dirty = False while reduce_queue: - LOG.debug("Reduce queue: %r", reduce_queue) + LOG.debug( + "Reduce queue: %r", + ", ".join( + str(x.relative_to(self._testcase_root)) for x in reduce_queue + ), + ) file = reduce_queue.pop(0) file_no += 1 LOG.info( @@ -141,27 +142,31 @@ def __iter__(self): for reduction in self._current_reducer: reduction.dump() - testcases = TestCase.load(self._testcase_root, True) + testcases = [] + for test in sorted(self._testcase_root.iterdir()): + testcases.append(TestCase.load(test, catalog=True)) LOG.info("[%s] %s", self.name, self._current_reducer.description) yield testcases - if self._current_feedback: - testcase_root_dirty = False - else: + if not self._current_feedback: self._tried.add(self._calculate_testcase_hash()) - if self._current_feedback and self._current_served is not None: - testcases = TestCase.load(self._testcase_root, True) - try: - self.purge_unserved(testcases, self._current_served) - finally: - for testcase in testcases: - testcase.cleanup() - num_files_before = len(self._files_to_reduce) - LOG.debug("files being reduced before: %r", self._files_to_reduce) - self.rescan_files_to_reduce() - LOG.debug("files being reduced after: %r", self._files_to_reduce) + else: + LOG.debug( + "files being reduced before: %r", + ", ".join( + str(x.relative_to(self._testcase_root)) + for x in self._files_to_reduce + ), + ) + self.rescan_files_to_reduce(testcases) + LOG.debug( + "files being reduced after: %r", + ", ".join( + str(x.relative_to(self._testcase_root)) + for x in self._files_to_reduce + ), + ) files_to_reduce = set(self._files_to_reduce) reduce_queue = sorted(set(reduce_queue) & files_to_reduce) - testcase_root_dirty = len(self._files_to_reduce) != num_files_before if file not in files_to_reduce: # current reduction was for a purged file break @@ -169,14 +174,6 @@ def __iter__(self): # write out the best found testcase self._current_reducer.testcase.dump() self._current_reducer = None - if testcase_root_dirty: - # purging unserved files enabled us to exit early from the loop. - # need to yield once more to set this trimmed version to the current best - # in ReduceManager - testcases = TestCase.load(self._testcase_root, True) - LOG.info("[%s] final iteration triggered by purge_optional", self.name) - yield testcases - assert self._current_feedback, "Purging unserved files broke the testcase." class Check(_LithiumStrategy): diff --git a/grizzly/reduce/strategies/testcases.py b/grizzly/reduce/strategies/testcases.py index fe54611e..eb353792 100644 --- a/grizzly/reduce/strategies/testcases.py +++ b/grizzly/reduce/strategies/testcases.py @@ -4,6 +4,7 @@ """Grizzly reducer testcase list strategy definition.""" from logging import getLogger +from shutil import rmtree from ...common.storage import TestCase from . import Strategy @@ -38,13 +39,11 @@ def __init__(self, testcases): self._current_feedback = None self._current_served = None - def update(self, success, served=None): + def update(self, success): """Inform the strategy whether or not the last reduction yielded was good. Arguments: success (bool): Whether or not the last reduction was acceptable. - served (list(list(str))): The list of served files for each testcase in the - last reduction. Returns: None @@ -52,7 +51,6 @@ def update(self, success, served=None): assert self._current_feedback is None assert self._current_served is None self._current_feedback = success - self._current_served = served def __iter__(self): """Iterate over potential reductions of testcases according to this strategy. @@ -67,70 +65,34 @@ def __iter__(self): """ assert self._current_feedback is None idx = 0 - testcases = None - try: - testcases = TestCase.load(self._testcase_root, True) - n_testcases = len(testcases) - # indicates that self._testcase_root contains changes that haven't been - # yielded (if iteration ends, changes would be lost) - testcase_root_dirty = False - while True: - if n_testcases <= 1: - LOG.info( - "Testcase list has length %d, not enough to reduce!", - n_testcases, - ) - break - if idx >= n_testcases: - LOG.info("Attempted to remove every single testcase") - break - # try removing the testcase at idx - if testcases is None: - testcases = TestCase.load(self._testcase_root, True) - assert n_testcases == len(testcases) - testcases.pop(idx).cleanup() - yield testcases - testcases = None # caller owns testcases now - assert self._current_feedback is not None, "no feedback received!" - - if self._current_feedback: - testcase_root_dirty = False - LOG.info( - "Removing testcase %d/%d was successful!", idx + 1, n_testcases - ) - testcases = TestCase.load(self._testcase_root, True) - try: - # remove the actual testcase we were reducing - testcases.pop(idx).cleanup() - if testcases and self._current_served is not None: - testcase_root_dirty = self.purge_unserved( - testcases, self._current_served - ) - else: - self.dump_testcases(testcases, recreate_tcroot=True) - finally: - for testcase in testcases: - testcase.cleanup() - testcases = TestCase.load(self._testcase_root, True) - n_testcases = len(testcases) - else: - LOG.info("No result without testcase %d/%d", idx + 1, n_testcases) - idx += 1 - # reset - self._current_feedback = None - self._current_served = None - if testcase_root_dirty: - # purging unserved files enabled us to exit early from the loop. - # need to yield once more to set this trimmed version to the current - # best in ReduceManager - testcases = TestCase.load(self._testcase_root, True) - LOG.info("[%s] final iteration triggered by purge_optional", self.name) - yield testcases - testcases = None # caller owns testcases now - assert ( - self._current_feedback - ), "Purging unserved files broke the testcase." - finally: - if testcases is not None: - for testcase in testcases: - testcase.cleanup() + testcases = [] + for test in sorted(self._testcase_root.iterdir()): + testcases.append(TestCase.load(test)) + n_testcases = len(testcases) + while True: + if n_testcases <= 1: + LOG.info( + "Testcase list has length %d, not enough to reduce!", + n_testcases, + ) + break + if idx >= n_testcases: + LOG.info("Attempted to remove every single testcase") + break + # try removing the testcase at idx + excluded = testcases.pop(idx) + yield testcases + assert self._current_feedback is not None, "no feedback received!" + if self._current_feedback: + rmtree(excluded.root, ignore_errors=True) + LOG.info( + "Removing testcase %d/%d was successful!", idx + 1, n_testcases + ) + n_testcases = len(testcases) + else: + testcases.insert(idx, excluded) + LOG.info("No result without testcase %d/%d", idx + 1, n_testcases) + idx += 1 + # reset + self._current_feedback = None + self._current_served = None diff --git a/grizzly/reduce/test_main.py b/grizzly/reduce/test_main.py index 28d525c6..f69a2fec 100644 --- a/grizzly/reduce/test_main.py +++ b/grizzly/reduce/test_main.py @@ -2,13 +2,11 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """Unit tests for `grizzly.reduce.main`.""" -from unittest.mock import Mock - from pytest import mark, raises from ..common.storage import TestCase, TestCaseLoadFailure from ..common.utils import ConfigError, Exit -from ..target import AssetManager, Target, TargetLaunchError, TargetLaunchTimeout +from ..target import Target, TargetLaunchError, TargetLaunchTimeout from . import ReduceManager from .args import ReduceArgs, ReduceFuzzManagerIDArgs, ReduceFuzzManagerIDQualityArgs from .exceptions import GrizzlyReduceBaseException @@ -52,11 +50,15 @@ def test_args_02(tmp_path): ReduceArgs().parse_args([str(exe), str(inp), "--report-period", "15"]) -def test_args_03(tmp_path): +def test_args_03(tmp_path, capsys): """test ReduceFuzzManagerIDArgs""" exe = tmp_path / "binary" exe.touch() ReduceFuzzManagerIDArgs().parse_args([str(exe), "123"]) + # error cases + with raises(SystemExit): + ReduceFuzzManagerIDArgs().parse_args([str(exe), "123", "--no-harness"]) + assert "error: '--no-harness' requires '--test-index'" in capsys.readouterr()[-1] def test_args_04(tmp_path): @@ -74,7 +76,7 @@ def test_args_04(tmp_path): TargetLaunchError("error", None), None, {}, - Exit.ERROR, + Exit.LAUNCH_FAILURE, False, ), ( @@ -82,7 +84,7 @@ def test_args_04(tmp_path): TargetLaunchTimeout, None, {}, - Exit.ERROR, + Exit.LAUNCH_FAILURE, False, ), ( @@ -111,26 +113,18 @@ def test_args_04(tmp_path): ), ( "grizzly.reduce.core.ReplayManager.load_testcases", + ConfigError("", 999), None, - [], - {}, - Exit.ERROR, - False, - ), - ( - "grizzly.reduce.core.ReplayManager.load_testcases", - None, - ([Mock(hang=False), Mock(hang=False)], Mock(spec_set=AssetManager), {}), - {"no_harness": True}, - Exit.ARGS, + {"no_harness": False}, + 999, False, ), ( "grizzly.reduce.core.ReplayManager.load_testcases", - ConfigError("", 999), + RuntimeError, None, {}, - 999, + Exit.ERROR, False, ), ], @@ -141,6 +135,7 @@ def test_main_exit( """test ReduceManager.main() failure cases""" mocker.patch("grizzly.reduce.core.FuzzManagerReporter", autospec=True) mocker.patch("grizzly.reduce.core.load_plugin", autospec=True) + mocker.patch("grizzly.reduce.core.ReductionStatus", autospec=True) mocker.patch("grizzly.reduce.core.Sapphire", autospec=True) if use_sig: @@ -154,7 +149,7 @@ def test_main_exit( # setup args args = mocker.Mock( ignore=["fake"], - input=(tmp_path / "test.html"), + input=[tmp_path / "test.html"], min_crashes=1, relaunch=1, repeat=1, @@ -194,8 +189,9 @@ def test_main_launch_error(mocker, exc_type): # setup args args = mocker.Mock( ignore=["fake"], - input="test", + input=["test"], min_crashes=1, + no_harness=False, relaunch=1, repeat=1, sig=None, @@ -227,7 +223,7 @@ def test_main_https_support(mocker, tmp_path, https_supported): # setup args args = mocker.Mock( ignore=["fake"], - input=tmp_path / "test.html", + input=[tmp_path / "test.html"], min_crashes=1, relaunch=1, repeat=1, diff --git a/grizzly/reduce/test_reduce.py b/grizzly/reduce/test_reduce.py index 41f7374b..e842ebc2 100644 --- a/grizzly/reduce/test_reduce.py +++ b/grizzly/reduce/test_reduce.py @@ -291,49 +291,47 @@ def replay_run(_tests, _time_limit, **kw): log_path.mkdir(exist_ok=True) _fake_save_logs_foo(log_path) report = Report(log_path, Path("bin")) - results.append(ReplayResult(report, [["test.html"]], [], True)) + results.append(ReplayResult(report, [], True)) return results replayer.run.side_effect = replay_run - test = TestCase("test.html", "test-adapter") - test.add_from_bytes(b"1", "test.html") - tests = [test] + with TestCase("test.html", "test-adapter") as test: + test.add_from_bytes(b"1", file_name=test.entry_point) + test.dump(tmp_path / "src1", include_details=True) + tests = [test.load(tmp_path / "src1")] if harness_last_crashes is not None: - test = TestCase("test.html", "test-adapter") - test.add_from_bytes(b"2", "test.html") - tests.append(test.clone()) + with TestCase("test.html", "test-adapter") as test: + test.add_from_bytes(b"2", file_name=test.entry_point) + test.dump(tmp_path / "src2", include_details=True) + tests.append(test.load(tmp_path / "src2")) log_path = tmp_path / "logs" - try: - mgr = ReduceManager( - None, - mocker.Mock(spec_set=Sapphire, timeout=30), - mocker.Mock(spec_set=Target), - tests, - None, - log_path, - use_harness=use_harness, - ) + with ReduceManager( + None, + mocker.Mock(spec_set=Sapphire, timeout=30), + mocker.Mock(spec_set=Target), + tests, + None, + log_path, + use_harness=use_harness, + ) as mgr: repeat, min_crashes = mgr.run_reliability_analysis() - finally: - for test in tests: - test.cleanup() - - observed = { - "replay_iters": replayer.run.call_count, - "repeat": repeat, - "min_crashes": min_crashes, - "use_harness": mgr._use_harness, - "launch_iters": mgr._status.iterations, - } - expected = { - "replay_iters": expected_iters, - "repeat": expected_repeat, - "min_crashes": expected_min_crashes, - "use_harness": result_harness, - "launch_iters": expected_iters * 11, - } + + observed = { + "replay_iters": replayer.run.call_count, + "repeat": repeat, + "min_crashes": min_crashes, + "use_harness": mgr._use_harness, + "launch_iters": mgr._status.iterations, + } + expected = { + "replay_iters": expected_iters, + "repeat": expected_repeat, + "min_crashes": expected_min_crashes, + "use_harness": result_harness, + "launch_iters": expected_iters * 11, + } assert observed == expected @@ -554,37 +552,35 @@ def replay_run(testcases, _time_limit, **kw): else: _fake_save_logs_bar(log_path) report = Report(log_path, Path("bin")) - return [ReplayResult(report, [["test.html"]], [], expected)] + return [ReplayResult(report, [], expected)] return [] replayer.run.side_effect = replay_run - test = TestCase("test.html", "test-adapter") - test.add_from_bytes(original, "test.html") - tests = [test] + with TestCase("test.html", "test-adapter") as test: + test.add_from_bytes(original, file_name=test.entry_point) + test.dump(tmp_path / "src") + tests = [TestCase.load(tmp_path / "src")] + log_path = tmp_path / "logs" target = mocker.Mock(spec_set=Target) target.filtered_environ.return_value = {} target.asset_mgr = mocker.Mock(spec_set=AssetManager) - try: - mgr = ReduceManager( - [], - mocker.Mock(spec_set=Sapphire, timeout=30), - target, - tests, - strategies, - log_path, - use_analysis=False, - ) + with ReduceManager( + [], + mocker.Mock(spec_set=Sapphire, timeout=30), + target, + tests, + strategies, + log_path, + use_analysis=False, + ) as mgr: if isinstance(result, type) and issubclass(result, BaseException): with raises(result): mgr.run() else: assert mgr.run() == result - finally: - for test in tests: - test.cleanup() expected_dirs = set() if n_reports: @@ -624,14 +620,14 @@ def replay_run(_tests, _time_limit, **kw): log_path.mkdir() _fake_save_logs_foo(log_path) report = Report(log_path, Path("bin")) - return [ReplayResult(report, [["test.html"]], [], True)] + return [ReplayResult(report, [], True)] return [] replayer.run.side_effect = replay_run (tmp_path / "test.html").touch() - testcases = TestCase.load(tmp_path / "test.html") - assert testcases + testcase = TestCase.load(tmp_path / "test.html") + assert testcase log_path = tmp_path / "logs" fake_strat = mocker.MagicMock(spec_set=Strategy) @@ -641,7 +637,7 @@ def fake_iter(): for count_ in range(1, 61): LOG.debug("fake_iter() %d", count_) (tmp_path / "test.html").write_text(str(count_)) - testcases = TestCase.load(tmp_path / "test.html") + testcases = [TestCase.load(tmp_path / "test.html")] assert testcases yield testcases @@ -651,21 +647,17 @@ def fake_iter(): target = mocker.Mock(spec_set=Target) target.filtered_environ.return_value = {} target.asset_mgr = mocker.Mock(spec_set=AssetManager) - try: - mgr = ReduceManager( - [], - mocker.Mock(spec_set=Sapphire, timeout=30), - target, - testcases, - ["fake"], - log_path, - use_analysis=False, - report_period=30, - ) + with ReduceManager( + [], + mocker.Mock(spec_set=Sapphire, timeout=30), + target, + [testcase], + ["fake"], + log_path, + use_analysis=False, + report_period=30, + ) as mgr: assert mgr.run() == 0 - finally: - for test in testcases: - test.cleanup() # should be 2 reports: one made at time=30 (for crash on 20th iter), # and one at time=60 (for crash on 40th iter) @@ -695,14 +687,14 @@ def replay_run(_tests, _time_limit, **kw): log_path.mkdir() _fake_save_logs_foo(log_path) report = Report(log_path, Path("bin")) - return [ReplayResult(report, [["test.html"]], [], True)] + return [ReplayResult(report, [], True)] return [] replayer.run.side_effect = replay_run (tmp_path / "test.html").touch() - testcases = TestCase.load(tmp_path / "test.html") - assert testcases + testcase = TestCase.load(tmp_path / "test.html") + assert testcase log_path = tmp_path / "logs" fake_strat = mocker.MagicMock(spec_set=Strategy) @@ -712,7 +704,7 @@ def fake_iter(): for count_ in range(1, 31): LOG.debug("fake_iter() %d", count_) (tmp_path / "test.html").write_text(str(count_)) - testcases = TestCase.load(tmp_path / "test.html") + testcases = [TestCase.load(tmp_path / "test.html")] assert testcases yield testcases raise KeyboardInterrupt() @@ -723,21 +715,18 @@ def fake_iter(): target = mocker.Mock(spec_set=Target) target.filtered_environ.return_value = {} target.asset_mgr = mocker.Mock(spec_set=AssetManager) - try: - mgr = ReduceManager( - [], - mocker.Mock(spec_set=Sapphire, timeout=30), - target, - testcases, - ["fake"], - log_path, - use_analysis=False, - ) + + with ReduceManager( + [], + mocker.Mock(spec_set=Sapphire, timeout=30), + target, + [testcase], + ["fake"], + log_path, + use_analysis=False, + ) as mgr: with raises(KeyboardInterrupt): mgr.run() - finally: - for test in testcases: - test.cleanup() n_reports = 1 reports = {"20"} @@ -767,14 +756,14 @@ def replay_run(testcases, _time_limit, **kw): log_path.mkdir() _fake_save_logs_foo(log_path) report = Report(log_path, Path("bin")) - return [ReplayResult(report, [["test.html"]], [], True)] + return [ReplayResult(report, [], True)] return [] replayer.run.side_effect = replay_run (tmp_path / "test.html").write_text("123\n") - testcases = TestCase.load(tmp_path / "test.html") - assert testcases + testcase = TestCase.load(tmp_path / "test.html") + assert testcase log_path = tmp_path / "logs" mocker.patch("grizzly.common.reporter.Collector", autospec=True) @@ -784,21 +773,17 @@ def replay_run(testcases, _time_limit, **kw): target = mocker.Mock(spec_set=Target) target.filtered_environ.return_value = {} target.asset_mgr = mocker.Mock(spec_set=AssetManager) - try: - mgr = ReduceManager( - [], - mocker.Mock(spec_set=Sapphire, timeout=30), - target, - testcases, - ["check", "lines"], - log_path, - use_analysis=False, - report_to_fuzzmanager=True, - ) + with ReduceManager( + [], + mocker.Mock(spec_set=Sapphire, timeout=30), + target, + [testcase], + ["check", "lines"], + log_path, + use_analysis=False, + report_to_fuzzmanager=True, + ) as mgr: assert mgr.run() == 0 - finally: - for test in testcases: - test.cleanup() assert reporter.return_value.submit.call_count == 1 report_args, _ = reporter.return_value.submit.call_args @@ -830,14 +815,14 @@ def replay_run(testcases, _time_limit, **kw): log_path.mkdir() _fake_save_logs_foo(log_path) report = Report(log_path, Path("bin")) - return [ReplayResult(report, [["test.html"]], [], True)] + return [ReplayResult(report, [], True)] return [] replayer.run.side_effect = replay_run (tmp_path / "test.html").write_text("123\n") - testcases = TestCase.load(tmp_path / "test.html") - assert testcases + testcase = TestCase.load(tmp_path / "test.html") + assert testcase log_path = tmp_path / "logs" reporter = mocker.patch("grizzly.reduce.core.FilesystemReporter", autospec=True) @@ -849,7 +834,6 @@ def submit(test_cases, report, force=False): for test in test_cases: assert test.assets.get("example") assert test.env_vars == {"test": "abc"} - test.cleanup() reporter.return_value.submit.side_effect = submit @@ -859,20 +843,16 @@ def submit(test_cases, report, force=False): (tmp_path / "example_asset").touch() asset_mgr.add("example", tmp_path / "example_asset", copy=False) target.asset_mgr = asset_mgr - try: - mgr = ReduceManager( - [], - mocker.Mock(spec_set=Sapphire, timeout=30), - target, - testcases, - ["check", "lines"], - log_path, - use_analysis=False, - ) + with ReduceManager( + [], + mocker.Mock(spec_set=Sapphire, timeout=30), + target, + [testcase], + ["check", "lines"], + log_path, + use_analysis=False, + ) as mgr: assert mgr.run() == 0 - finally: - for test in testcases: - test.cleanup() assert reporter.return_value.submit.call_count == 1 @@ -968,39 +948,34 @@ def replay_run(_testcases, _time_limit, **kw): log_path.mkdir() _fake_save_logs_foo(log_path) report = Report(log_path, Path("bin")) - return [ReplayResult(report, [["test.html"]], durations, interesting)] + return [ReplayResult(report, durations, interesting)] replayer.run.side_effect = replay_run - test = TestCase("test.html", "test-adapter") - test.add_from_bytes(b"123\n", "test.html") - tests = [test] + (tmp_path / "test.html").touch() + test = TestCase.load(tmp_path / "test.html") log_path = tmp_path / "logs" target = mocker.Mock(spec_set=Target) target.filtered_environ.return_value = {} target.asset_mgr = mocker.Mock(spec_set=AssetManager) server = mocker.Mock(spec_set=Sapphire, timeout=iter_input) - try: - mgr = ReduceManager( - [], - server, - target, - tests, - ["check"], - log_path, - use_analysis=False, - idle_delay=idle_input, - static_timeout=static_timeout, - ) + with ReduceManager( + [], + server, + target, + [test], + ["check"], + log_path, + use_analysis=False, + idle_delay=idle_input, + static_timeout=static_timeout, + ) as mgr: if isinstance(result, type) and issubclass(result, BaseException): with raises(result): mgr.run() else: assert mgr.run() == result - finally: - for test in tests: - test.cleanup() assert server.timeout == iter_output assert mgr._idle_delay == idle_output diff --git a/grizzly/reduce/test_strategies.py b/grizzly/reduce/test_strategies.py index 96b2bea7..a9e43970 100644 --- a/grizzly/reduce/test_strategies.py +++ b/grizzly/reduce/test_strategies.py @@ -26,27 +26,31 @@ @mark.parametrize("is_hang", [True, False]) -def test_strategy_tc_load(is_hang): +def test_strategy_tc_load(tmp_path, is_hang): """test that strategy base class dump and load doesn't change testcase metadata""" class _TestStrategy(Strategy): def __iter__(self): - yield TestCase.load(self._testcase_root, False) + testcases = [] + for test in sorted(self._testcase_root.iterdir()): + testcases.append(TestCase.load(test)) + yield testcases - def update(self, success, served=None): + def update(self, success): pass - # create testcase that is_hang - with TestCase("a.htm", "adpt", input_fname="fn", time_limit=2) as src: - src.duration = 1.2 - src.hang = is_hang - src.add_from_bytes(b"123", "a.htm") - strategy = _TestStrategy([src]) - for attempt in strategy: - assert len(attempt) == 1 - assert attempt[0].hang == is_hang - attempt[0].cleanup() - strategy.update(False) + # create testcase + with TestCase("a.htm", "adpt", input_fname="fn", time_limit=2) as test: + test.duration = 1.2 + test.hang = is_hang + test.add_from_bytes(b"123", test.entry_point) + test.dump(tmp_path / "src", include_details=True) + + with _TestStrategy([TestCase.load(tmp_path / "src")]) as strategy: + for attempt in strategy: + assert len(attempt) == 1 + assert attempt[0].hang == is_hang + strategy.update(False) def _fake_save_logs_foo(result_logs): @@ -164,37 +168,32 @@ def replay_run(testcases, _time_limit, **kw): log_path.mkdir() _fake_save_logs_foo(log_path) report = Report(log_path, Path("bin")) - return [ - ReplayResult(report, [["test.html"]] * len(testcases), [], True) - ] + return [ReplayResult(report, [], True)] return [] replayer.run.side_effect = replay_run tests = [] - for data in test_data: - test = TestCase("test.html", "test-adapter") - test.add_from_bytes(data, "test.html") - tests.append(test) + for num, data in enumerate(test_data): + with TestCase("test.html", "test-adapter") as test: + test.add_from_bytes(data, file_name=test.entry_point) + test.dump(tmp_path / "src" / f"{num:02d}", include_details=True) + tests.append(TestCase.load(tmp_path / "src" / f"{num:02d}")) log_path = tmp_path / "logs" target = mocker.Mock(spec_set=Target) target.filtered_environ.return_value = {} target.asset_mgr = mocker.Mock(spec_set=AssetManager) - try: - mgr = ReduceManager( - [], - mocker.Mock(spec_set=Sapphire, timeout=30), - target, - tests, - strategies, - log_path, - use_analysis=False, - ) + with ReduceManager( + [], + mocker.Mock(spec_set=Sapphire, timeout=30), + target, + tests, + strategies, + log_path, + use_analysis=False, + ) as mgr: assert mgr.run() == 0 - finally: - for test in tests: - test.cleanup() assert replayer.run.call_count == expected_run_calls assert set(log_path.iterdir()) == {log_path / "reports"} @@ -205,193 +204,6 @@ def replay_run(testcases, _time_limit, **kw): ), list((log_path / "reports").iterdir()) -PurgeUnservedTestParams = namedtuple( - "PurgeUnservedTestParams", - "strategies, test_data, served, expected_results, expected_run_calls," - "expected_num_reports, purging_breaks", -) - - -@mark.parametrize( - PurgeUnservedTestParams._fields, - [ - # single test, first reduction uses 2 files, second uses only target file. - PurgeUnservedTestParams( - strategies=["chars"], - test_data=[{"test.html": b"123", "opt.html": b"456"}], - served=[[["test.html", "opt.html"]], [["test.html"]], [["test.html"]]], - expected_results={"1"}, - expected_run_calls=5, - expected_num_reports=2, - purging_breaks=False, - ), - # single test, first reduction uses target only - PurgeUnservedTestParams( - strategies=["chars"], - test_data=[{"test.html": b"123", "opt.html": b"456"}], - served=[[["test.html"]], [["test.html"]]], - expected_results={"1"}, - expected_run_calls=3, - expected_num_reports=2, - purging_breaks=False, - ), - # single test, first reduction uses 2 files, second uses only optional file. - # (no results -> Assertion) - PurgeUnservedTestParams( - strategies=["chars"], - test_data=[{"test.html": b"123", "opt.html": b"456"}], - served=[[["test.html", "opt.html"]], [["opt.html"]]], - expected_results=set(), - expected_run_calls=4, - expected_num_reports=None, - purging_breaks=True, - ), - # double test, first reduction uses all files, second uses only target file in - # second test. - PurgeUnservedTestParams( - strategies=["chars"], - test_data=[ - {"test.html": b"123", "opt.html": b"456"}, - {"test.html": b"789", "opt.html": b"abc"}, - ], - served=[ - [["test.html", "opt.html"], ["test.html", "opt.html"]], - [["test.html", "opt.html"], ["test.html"]], - [["test.html", "opt.html"], ["test.html"]], - ], - expected_results={"1", "4", "7"}, - expected_run_calls=6, - expected_num_reports=3, - purging_breaks=False, - ), - # double test, first reduction uses all files, second uses only optional file - # (first test remains) - PurgeUnservedTestParams( - strategies=["chars"], - test_data=[ - {"test.html": b"123", "opt.html": b"456"}, - {"test.html": b"789", "opt.html": b"abc"}, - ], - served=[ - [["test.html", "opt.html"], ["test.html", "opt.html"]], - [["test.html", "opt.html"], ["opt.html"]], - [["test.html", "opt.html"]], - ], - expected_results={"1", "4"}, - expected_run_calls=5, - expected_num_reports=2, - purging_breaks=False, - ), - # triple test, list strategy. first test gets reduced, third gets eliminated - PurgeUnservedTestParams( - strategies=["list"], - test_data=[ - {"test.html": b"123"}, - {"test.html": b"456"}, - {"test.html": b"789"}, - ], - served=[[["test.html"]], [["test.html"]], [["test.html"]]], - expected_results={"456"}, - expected_run_calls=2, - expected_num_reports=2, - purging_breaks=False, - ), - # triple test, list strategy. None for served still eliminates first two tests - PurgeUnservedTestParams( - strategies=["list"], - test_data=[ - {"test.html": b"123"}, - {"test.html": b"456"}, - {"test.html": b"789"}, - ], - served=[None, None, None], - expected_results={"789"}, - expected_run_calls=2, - expected_num_reports=2, - purging_breaks=False, - ), - ], -) -def test_purge_unserved( - mocker, - tmp_path, - strategies, - test_data, - served, - expected_results, - expected_run_calls, - expected_num_reports, - purging_breaks, -): - """test purging unserved files""" - mocker.patch("grizzly.reduce.strategies.lithium._contains_dd", return_value=True) - replayer = mocker.patch("grizzly.reduce.core.ReplayManager", autospec=True) - replayer = replayer.return_value - - def replay_run(testcases, _time_limit, **kw): - kw["on_iteration_cb"]() - # test.html and opt.html should always contain one line. - # return [] (no result) if either of them exist and are empty - has_any = False - for test in testcases: - for file in ("test.html", "opt.html"): - if file in test.contents: - LOG.debug("testcase contains %s", file) - has_any = True - contents = test.get_file(file).data_file.read_text() - if not contents.strip(): - return [] - if not has_any: - return [] - log_path = tmp_path / f"crash{replayer.run.call_count}_logs" - log_path.mkdir() - _fake_save_logs_foo(log_path) - report = Report(log_path, Path("bin")) - return [ReplayResult(report, served.pop(0), [], True)] - - replayer.run.side_effect = replay_run - - tests = [] - for testcase in test_data: - test = TestCase("test.html", "test-adapter") - for filename, data in testcase.items(): - test.add_from_bytes(data, filename) - tests.append(test) - log_path = tmp_path / "logs" - - target = mocker.Mock(spec_set=Target) - target.filtered_environ.return_value = {} - target.asset_mgr = mocker.Mock(spec_set=AssetManager) - try: - mgr = ReduceManager( - [], - mocker.Mock(spec_set=Sapphire, timeout=30), - target, - tests, - strategies, - log_path, - use_analysis=False, - ) - if purging_breaks: - with raises(AssertionError): - mgr.run() - else: - assert mgr.run() == 0 - finally: - for test in tests: - test.cleanup() - - assert replayer.run.call_count == expected_run_calls - if purging_breaks: - return - assert set(log_path.iterdir()) == {log_path / "reports"} - tests = {test.read_text() for test in log_path.glob("reports/*-*/*.html")} - assert tests == expected_results - assert ( - sum(1 for _ in (log_path / "reports").iterdir()) == expected_num_reports - ), list((log_path / "reports").iterdir()) - - def test_dd_only(mocker, tmp_path): """test that only files containing DDBEGIN/END are reduced""" replayer = mocker.patch("grizzly.reduce.core.ReplayManager", autospec=True) @@ -408,34 +220,32 @@ def replay_run(testcases, _time_limit, **kw): log_path.mkdir() _fake_save_logs_foo(log_path) report = Report(log_path, Path("bin")) - return [ReplayResult(report, [["test.html", "other.html"]], [], True)] + return [ReplayResult(report, [], True)] return [] replayer.run.side_effect = replay_run - test = TestCase("test.html", "test-adapter") - test.add_from_bytes(b"DDBEGIN\n123\nrequired\nDDEND\n", "test.html") - test.add_from_bytes(b"blah\n", "other.html") + with TestCase("test.html", "test-adapter") as test: + test.add_from_bytes(b"DDBEGIN\n123\nrequired\nDDEND\n", file_name="test.html") + test.add_from_bytes(b"blah\n", file_name="other.html") + test.dump(tmp_path / "src", include_details=True) + test = TestCase.load(tmp_path / "src", catalog=True) tests = [test] log_path = tmp_path / "logs" target = mocker.Mock(spec_set=Target) target.filtered_environ.return_value = {} target.asset_mgr = mocker.Mock(spec_set=AssetManager) - try: - mgr = ReduceManager( - [], - mocker.Mock(spec_set=Sapphire, timeout=30), - target, - tests, - ["lines"], - log_path, - use_analysis=False, - ) + with ReduceManager( + [], + mocker.Mock(spec_set=Sapphire, timeout=30), + target, + tests, + ["lines"], + log_path, + use_analysis=False, + ) as mgr: assert mgr.run() == 0 - finally: - for test in tests: - test.cleanup() expected_run_calls = 3 expected_results = {"DDBEGIN\nrequired\nDDEND\n"} @@ -443,6 +253,7 @@ def replay_run(testcases, _time_limit, **kw): assert replayer.run.call_count == expected_run_calls assert set(log_path.iterdir()) == {log_path / "reports"} + assert len(tuple(log_path.glob("reports/*-*/*.html"))) == 2 tests = {test.read_text() for test in log_path.glob("reports/*-*/test.html")} assert tests == expected_results assert ( diff --git a/grizzly/reduce/test_strategies_beautify.py b/grizzly/reduce/test_strategies_beautify.py index daf4a58f..0d480e2e 100644 --- a/grizzly/reduce/test_strategies_beautify.py +++ b/grizzly/reduce/test_strategies_beautify.py @@ -13,12 +13,13 @@ LOG = getLogger(__name__) -def _test_beautify(cls, interesting, test_name, test_data, reduced, mocker): +def _test_beautify(cls, interesting, test_name, test_data, reduced, mocker, tmp_path): mocker.patch("grizzly.reduce.strategies.beautify._contains_dd", return_value=True) - best_test = TestCase(test_name, "test-adapter") - best_test.add_from_bytes(test_data.encode("ascii"), test_name) - best_tests = [best_test] + with TestCase(test_name, "test-adapter") as test: + test.add_from_bytes(test_data.encode("ascii"), test.entry_point) + test.dump(tmp_path / "src", include_details=True) + best_tests = [TestCase.load(tmp_path / "src")] def _interesting(testcases): for test in testcases: @@ -30,17 +31,12 @@ def _interesting(testcases): try: with cls(best_tests) as sgy: for tests in sgy: - try: - result = _interesting(tests) - sgy.update(result) - if result: - for test in best_tests: - test.cleanup() - best_tests = tests - tests = [] - finally: - for test in tests: + result = _interesting(tests) + sgy.update(result) + if result: + for test in best_tests: test.cleanup() + best_tests = [x.clone() for x in tests] assert len(best_tests) == 1 contents = ( best_tests[0].get_file(test_name).data_file.read_bytes().decode("ascii") @@ -86,9 +82,9 @@ def _interesting(testcases): ), ], ) -def test_beautify_js_1(test_data, reduced, mocker): +def test_beautify_js_1(test_data, reduced, mocker, tmp_path): _test_beautify( - JSBeautify, lambda x: "R" in x, "test.js", test_data, reduced, mocker + JSBeautify, lambda x: "R" in x, "test.js", test_data, reduced, mocker, tmp_path ) @@ -102,9 +98,15 @@ def test_beautify_js_1(test_data, reduced, mocker): ), ], ) -def test_beautify_js_2(test_data, reduced, mocker): +def test_beautify_js_2(test_data, reduced, mocker, tmp_path): _test_beautify( - JSBeautify, lambda x: "'R'+'R'" in x, "test.js", test_data, reduced, mocker + JSBeautify, + lambda x: "'R'+'R'" in x, + "test.js", + test_data, + reduced, + mocker, + tmp_path, ) @@ -161,9 +163,15 @@ def test_beautify_js_2(test_data, reduced, mocker): ), ], ) -def test_beautify_js_3(test_data, reduced, mocker): +def test_beautify_js_3(test_data, reduced, mocker, tmp_path): _test_beautify( - JSBeautify, lambda x: "R" in x, "test.html", test_data, reduced, mocker + JSBeautify, + lambda x: "R" in x, + "test.html", + test_data, + reduced, + mocker, + tmp_path, ) @@ -178,7 +186,7 @@ def test_beautify_js_3(test_data, reduced, mocker): ), ], ) -def test_beautify_js_4(test_data, reduced, mocker): +def test_beautify_js_4(test_data, reduced, mocker, tmp_path): _test_beautify( JSBeautify, lambda x: "Q" in x and "R" in x, @@ -186,6 +194,7 @@ def test_beautify_js_4(test_data, reduced, mocker): test_data, reduced, mocker, + tmp_path, ) @@ -214,9 +223,15 @@ def test_beautify_js_4(test_data, reduced, mocker): ), ], ) -def test_beautify_css_1(test_data, reduced, mocker): +def test_beautify_css_1(test_data, reduced, mocker, tmp_path): _test_beautify( - CSSBeautify, lambda x: "R" in x, "test.css", test_data, reduced, mocker + CSSBeautify, + lambda x: "R" in x, + "test.css", + test_data, + reduced, + mocker, + tmp_path, ) @@ -245,9 +260,15 @@ def test_beautify_css_1(test_data, reduced, mocker): ), ], ) -def test_beautify_css_2(test_data, reduced, mocker): +def test_beautify_css_2(test_data, reduced, mocker, tmp_path): _test_beautify( - CSSBeautify, lambda x: "R" in x, "test.html", test_data, reduced, mocker + CSSBeautify, + lambda x: "R" in x, + "test.html", + test_data, + reduced, + mocker, + tmp_path, ) @@ -258,9 +279,15 @@ def test_beautify_css_2(test_data, reduced, mocker): (CSSBeautify, "