From 6ba513f8ed73be09d878c86de41968bf4114ebec Mon Sep 17 00:00:00 2001 From: Zixian Cai <2891235+caizixian@users.noreply.github.com> Date: Mon, 20 Nov 2023 17:09:32 +1100 Subject: [PATCH] Add config for DaCapo Chopin (#128) --- .vscode/settings.json | 3 + docs/src/changelog.md | 4 +- src/running/__main__.py | 16 +- src/running/__version__.py | 2 +- src/running/benchmark.py | 101 ++++++----- src/running/command/fillin.py | 13 +- src/running/command/genadvice.py | 22 +-- src/running/command/log_preprocessor.py | 37 ++-- src/running/command/minheap.py | 58 ++++-- src/running/command/runbms.py | 136 +++++++++----- src/running/config.py | 42 ++--- src/running/config/base/dacapo.yml | 224 +++++++----------------- src/running/modifier.py | 22 ++- src/running/plugin/runbms/__init__.py | 59 +++++-- src/running/plugin/runbms/copyfile.py | 49 +++++- src/running/plugin/runbms/zulip.py | 77 +++++--- src/running/runtime.py | 40 +++-- src/running/suite.py | 211 ++++++++++++---------- src/running/util.py | 56 +++--- tests/test_config.py | 75 ++++---- tests/test_fillin.py | 7 +- tests/test_log_preprocessor.py | 60 ++++--- tests/test_modifier.py | 132 ++++++-------- tests/test_runbms.py | 7 +- tests/test_suite.py | 145 ++++++++------- tests/test_util.py | 48 +++-- 26 files changed, 889 insertions(+), 757 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..6bd93e6 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "env/bin/python" +} \ No newline at end of file diff --git a/docs/src/changelog.md b/docs/src/changelog.md index 62568cd..1575f36 100644 --- a/docs/src/changelog.md +++ b/docs/src/changelog.md @@ -1,6 +1,8 @@ # Changelog ## Unreleased ### Added +#### Benchmark Suites +- `DaCapo`: add 23.11-Chopin release and minheap values. ### Changed @@ -200,7 +202,7 @@ - Whitespaces can be used in config strings for visual alignment. They are ignored when parsed. #### Commands -- The `--slice` argument of `runbms` now accepts multiple comma-separated floating point numbers. +- The `--slice` argument of `runbms` now accepts multiple comma-separated floating point numbers. ### Removed #### Base Configurations diff --git a/src/running/__main__.py b/src/running/__main__.py index c111fa1..6eb716d 100644 --- a/src/running/__main__.py +++ b/src/running/__main__.py @@ -15,12 +15,13 @@ def setup_parser(): parser = argparse.ArgumentParser() - parser.add_argument("-v", "--verbose", action="store_true", - help="change logging level to DEBUG") - parser.add_argument("--version", action="version", - version="running {}".format(__VERSION__)) - parser.add_argument("-d", "--dry-run", action="store_true", - help="dry run") + parser.add_argument( + "-v", "--verbose", action="store_true", help="change logging level to DEBUG" + ) + parser.add_argument( + "--version", action="version", version="running {}".format(__VERSION__) + ) + parser.add_argument("-d", "--dry-run", action="store_true", help="dry run") subparsers = parser.add_subparsers() for m in MODULES: m.setup_parser(subparsers) @@ -38,7 +39,8 @@ def main(): log_level = logging.INFO logging.basicConfig( format="[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d %(message)s", - level=log_level) + level=log_level, + ) if args.get("dry_run") == True: set_dry_run(True) diff --git a/src/running/__version__.py b/src/running/__version__.py index 586df66..8b91166 100644 --- a/src/running/__version__.py +++ b/src/running/__version__.py @@ -1,2 +1,2 @@ VERSION = (0, 4, 2) -__VERSION__ = '.'.join(map(str, VERSION)) +__VERSION__ = ".".join(map(str, VERSION)) diff --git a/src/running/benchmark.py b/src/running/benchmark.py index 8f29742..150b2ed 100644 --- a/src/running/benchmark.py +++ b/src/running/benchmark.py @@ -22,11 +22,23 @@ class SubprocessrExit(Enum): Dryrun = 4 -B = TypeVar('B', bound='Benchmark') +B = TypeVar("B", bound="Benchmark") class Benchmark(object): - def __init__(self, suite_name: str, name: str, wrapper: Optional[str] = None, timeout: Optional[int] = None, override_cwd: Optional[Path] = None, companion: Optional[str] = None, runtime_specific_modifiers_strategy: Optional[Callable[[Runtime], Sequence[Modifier]]] = None, **kwargs): + def __init__( + self, + suite_name: str, + name: str, + wrapper: Optional[str] = None, + timeout: Optional[int] = None, + override_cwd: Optional[Path] = None, + companion: Optional[str] = None, + runtime_specific_modifiers_strategy: Optional[ + Callable[[Runtime], Sequence[Modifier]] + ] = None, + **kwargs + ): self.name = name self.suite_name = suite_name self.env_args: Dict[str, str] @@ -44,18 +56,20 @@ def __init__(self, suite_name: str, name: str, wrapper: Optional[str] = None, ti # ignore the current working directory provided by commands like runbms or minheap # certain benchmarks expect to be invoked from certain directories self.override_cwd = override_cwd - self.runtime_specific_modifiers_strategy: Callable[[ - Runtime], Sequence[Modifier]] + self.runtime_specific_modifiers_strategy: Callable[ + [Runtime], Sequence[Modifier] + ] if runtime_specific_modifiers_strategy is not None: - self.runtime_specific_modifiers_strategy = runtime_specific_modifiers_strategy + self.runtime_specific_modifiers_strategy = ( + runtime_specific_modifiers_strategy + ) else: self.runtime_specific_modifiers_strategy = lambda _runtime: [] def get_env_str(self) -> str: - return " ".join([ - "{}={}".format(k, smart_quote(v)) - for (k, v) in self.env_args.items() - ]) + return " ".join( + ["{}={}".format(k, smart_quote(v)) for (k, v) in self.env_args.items()] + ) def get_full_args(self, _runtime: Runtime) -> List[Union[str, Path]]: # makes a copy because the subclass might change the list @@ -83,18 +97,19 @@ def attach_modifiers(self: B, modifiers: Sequence[Modifier]) -> B: def to_string(self, runtime: Runtime) -> str: return "{} {}".format( self.get_env_str(), - " ".join([ - smart_quote(os.path.expandvars(x)) - for x in self.get_full_args(runtime) - ]) + " ".join( + [ + smart_quote(os.path.expandvars(x)) + for x in self.get_full_args(runtime) + ] + ), ) - def run(self, runtime: Runtime, cwd: Optional[Path] = None) -> Tuple[bytes, bytes, SubprocessrExit]: + def run( + self, runtime: Runtime, cwd: Optional[Path] = None + ) -> Tuple[bytes, bytes, SubprocessrExit]: if suite.is_dry_run(): - print( - self.to_string(runtime), - file=sys.stderr - ) + print(self.to_string(runtime), file=sys.stderr) return b"", b"", SubprocessrExit.Dryrun else: cmd = self.get_full_args(runtime) @@ -105,7 +120,8 @@ def run(self, runtime: Runtime, cwd: Optional[Path] = None) -> Tuple[bytes, byte stdout: Optional[bytes] if self.companion: companion_p = subprocess.Popen( - self.companion, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + self.companion, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) sleep(COMPANION_WAIT_START) try: p = subprocess.run( @@ -114,7 +130,7 @@ def run(self, runtime: Runtime, cwd: Optional[Path] = None) -> Tuple[bytes, byte stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=self.timeout, - cwd=self.override_cwd if self.override_cwd else cwd + cwd=self.override_cwd if self.override_cwd else cwd, ) subprocess_exit = SubprocessrExit.Normal stdout = p.stdout @@ -127,12 +143,12 @@ def run(self, runtime: Runtime, cwd: Optional[Path] = None) -> Tuple[bytes, byte finally: if self.companion: try: - companion_stdout, _ = companion_p.communicate( - timeout=10) + companion_stdout, _ = companion_p.communicate(timeout=10) companion_out += companion_stdout except subprocess.TimeoutExpired: logging.warning( - "Companion program not exited after 10 seconds timeout. Trying to kill ...") + "Companion program not exited after 10 seconds timeout. Trying to kill ..." + ) try: companion_p.kill() except PermissionError: @@ -153,7 +169,7 @@ def __init__(self, program: Path, program_args: List[Union[str, Path]], **kwargs def __str__(self) -> str: return self.to_string(DummyRuntime("")) - def attach_modifiers(self, modifiers: Sequence[Modifier]) -> 'BinaryBenchmark': + def attach_modifiers(self, modifiers: Sequence[Modifier]) -> "BinaryBenchmark": bb = super().attach_modifiers(modifiers) for m in modifiers: if not m.should_attach(self.suite_name, self.name): @@ -163,11 +179,9 @@ def attach_modifiers(self, modifiers: Sequence[Modifier]) -> 'BinaryBenchmark': elif type(m) == JVMArg: logging.warning("JVMArg not respected by BinaryBenchmark") elif isinstance(m, JVMClasspathAppend) or type(m) == JVMClasspathPrepend: - logging.warning( - "JVMClasspath not respected by BinaryBenchmark") + logging.warning("JVMClasspath not respected by BinaryBenchmark") elif type(m) == JSArg: - logging.warning( - "JSArg not respected by BinaryBenchmark") + logging.warning("JSArg not respected by BinaryBenchmark") return bb def get_full_args(self, _runtime: Runtime) -> List[Union[str, Path]]: @@ -178,7 +192,9 @@ def get_full_args(self, _runtime: Runtime) -> List[Union[str, Path]]: class JavaBenchmark(Benchmark): - def __init__(self, jvm_args: List[str], program_args: List[str], cp: List[str], **kwargs): + def __init__( + self, jvm_args: List[str], program_args: List[str], cp: List[str], **kwargs + ): super().__init__(**kwargs) self.jvm_args = jvm_args self.program_args = program_args @@ -190,7 +206,7 @@ def get_classpath_args(self) -> List[str]: def __str__(self) -> str: return self.to_string(DummyRuntime("java")) - def attach_modifiers(self, modifiers: Sequence[Modifier]) -> 'JavaBenchmark': + def attach_modifiers(self, modifiers: Sequence[Modifier]) -> "JavaBenchmark": jb = super().attach_modifiers(modifiers) for m in modifiers: if not m.should_attach(self.suite_name, self.name): @@ -204,8 +220,7 @@ def attach_modifiers(self, modifiers: Sequence[Modifier]) -> 'JavaBenchmark': elif type(m) == JVMClasspathPrepend: jb.cp = m.val + jb.cp elif type(m) == JSArg: - logging.warning( - "JSArg not respected by JavaBenchmark") + logging.warning("JSArg not respected by JavaBenchmark") return jb def get_full_args(self, runtime: Runtime) -> List[Union[str, Path]]: @@ -218,7 +233,9 @@ def get_full_args(self, runtime: Runtime) -> List[Union[str, Path]]: class JavaScriptBenchmark(Benchmark): - def __init__(self, js_args: List[str], program: str, program_args: List[str], **kwargs): + def __init__( + self, js_args: List[str], program: str, program_args: List[str], **kwargs + ): super().__init__(**kwargs) self.js_args = js_args self.program = program @@ -227,7 +244,7 @@ def __init__(self, js_args: List[str], program: str, program_args: List[str], ** def __str__(self) -> str: return self.to_string(DummyRuntime("js")) - def attach_modifiers(self, modifiers: Sequence[Modifier]) -> 'JavaScriptBenchmark': + def attach_modifiers(self, modifiers: Sequence[Modifier]) -> "JavaScriptBenchmark": jb = super().attach_modifiers(modifiers) for m in modifiers: if not m.should_attach(self.suite_name, self.name): @@ -237,8 +254,7 @@ def attach_modifiers(self, modifiers: Sequence[Modifier]) -> 'JavaScriptBenchmar elif type(m) == JVMArg: logging.warning("JVMArg not respected by JavaScriptBenchmark") elif isinstance(m, JVMClasspathAppend) or type(m) == JVMClasspathPrepend: - logging.warning( - "JVMClasspath not respected by JavaScriptBenchmark") + logging.warning("JVMClasspath not respected by JavaScriptBenchmark") elif type(m) == JSArg: jb.js_args.extend(m.val) return jb @@ -255,14 +271,19 @@ def get_full_args(self, runtime: Runtime) -> List[Union[str, Path]]: elif isinstance(runtime, SpiderMonkey): pass else: - raise TypeError("{} is of type {}, and not a valid runtime for JavaScriptBenchmark".format( - runtime, type(runtime))) + raise TypeError( + "{} is of type {}, and not a valid runtime for JavaScriptBenchmark".format( + runtime, type(runtime) + ) + ) cmd.extend(self.program_args) return cmd class JuliaBenchmark(Benchmark): - def __init__(self, julia_args: List[str], suite_path: Path, program_args: List[str], **kwargs): + def __init__( + self, julia_args: List[str], suite_path: Path, program_args: List[str], **kwargs + ): super().__init__(**kwargs) self.julia_args = julia_args self.suite_path = suite_path @@ -271,7 +292,7 @@ def __init__(self, julia_args: List[str], suite_path: Path, program_args: List[s def __str__(self) -> str: return self.to_string(DummyRuntime("julia")) - def attach_modifiers(self, modifiers: Sequence[Modifier]) -> 'JuliaBenchmark': + def attach_modifiers(self, modifiers: Sequence[Modifier]) -> "JuliaBenchmark": jb = super().attach_modifiers(modifiers) for m in modifiers: if type(m) == JuliaArg: diff --git a/src/running/command/fillin.py b/src/running/command/fillin.py index 293c1ea..fbe9941 100644 --- a/src/running/command/fillin.py +++ b/src/running/command/fillin.py @@ -2,7 +2,11 @@ import subprocess -def fillin(callback: Callable[[int, Iterable[int]], None], levels: int, start: Optional[int] = None): +def fillin( + callback: Callable[[int, Iterable[int]], None], + levels: int, + start: Optional[int] = None, +): """Fill the parameter space The parameter space is from 0, 1, 2, ..., 2^levels (not right-inclusive). @@ -23,7 +27,7 @@ def fillin(callback: Callable[[int, Iterable[int]], None], levels: int, start: O """ commenced = False if start is None: - callback(2**levels, range(0, 2 ** levels + 1, 2**(levels-1))) + callback(2**levels, range(0, 2**levels + 1, 2 ** (levels - 1))) commenced = True i = 1 while i < levels: @@ -32,7 +36,7 @@ def fillin(callback: Callable[[int, Iterable[int]], None], levels: int, start: O if start is not None and base == start: commenced = True if commenced: - callback(2**levels, range(base, 2 ** levels, step)) + callback(2**levels, range(base, 2**levels, step)) i += 1 @@ -43,6 +47,7 @@ def callback(end, ns): cmd.extend(map(str, ns)) output = subprocess.check_output(cmd) print(output.decode("utf-8"), end="") + return callback @@ -51,7 +56,7 @@ def setup_parser(subparsers): f.set_defaults(which="fillin") f.add_argument("PROG") f.add_argument("LEVELS", type=int) - f.add_argument("START", type=int, nargs='?', default=None) + f.add_argument("START", type=int, nargs="?", default=None) def run(args): diff --git a/src/running/command/genadvice.py b/src/running/command/genadvice.py index 5c1aefb..a0abc5b 100644 --- a/src/running/command/genadvice.py +++ b/src/running/command/genadvice.py @@ -12,8 +12,12 @@ advice_folder = sys.argv[1] -JikesRVM_HEADER = "============================ MMTk Statistics Totals ============================" -JikesRVM_FOOTER = "------------------------------ End MMTk Statistics -----------------------------" +JikesRVM_HEADER = ( + "============================ MMTk Statistics Totals ============================" +) +JikesRVM_FOOTER = ( + "------------------------------ End MMTk Statistics -----------------------------" +) def extract_blocks(lines, header, footer): @@ -33,7 +37,7 @@ def extract_blocks(lines, header, footer): def cleanse(filename): - sed_pattern = 's/{urls[^}]*}//g' + sed_pattern = "s/{urls[^}]*}//g" if sys.platform == "linux" or sys.platform == "linux2": cmd = ["sed", "-i", sed_pattern, filename] elif sys.platform == "darwin": @@ -46,11 +50,9 @@ def select_best_invocation(scenario): filename = "{}.log.gz".format(scenario) metrics = [] with gzip.open(os.path.join(advice_folder, filename)) as log_file: - stats_blocks = extract_blocks( - log_file, JikesRVM_HEADER, JikesRVM_FOOTER) + stats_blocks = extract_blocks(log_file, JikesRVM_HEADER, JikesRVM_FOOTER) for stats_block in stats_blocks: - stats = dict(zip(stats_block[0].split( - "\t"), stats_block[1].split("\t"))) + stats = dict(zip(stats_block[0].split("\t"), stats_block[1].split("\t"))) metrics.append(float(stats["time.gc"]) + float(stats["time.mu"])) if not metrics: print("No metric is found") @@ -76,14 +78,12 @@ def select_advice_file(scenario, best_invocation): def main(): scenario_logs = glob.glob(os.path.join(advice_folder, "*.log.gz")) - scenarios = [os.path.basename(s).replace(".log.gz", "") - for s in scenario_logs] + scenarios = [os.path.basename(s).replace(".log.gz", "") for s in scenario_logs] print("Found scenarios {}".format(scenarios)) for scenario in scenarios: print("Processing scenario {}".format(scenario)) best_invocation = select_best_invocation(scenario) - print("Best invocation for scenario {} is {}".format(scenario, - best_invocation)) + print("Best invocation for scenario {} is {}".format(scenario, best_invocation)) select_advice_file(scenario, best_invocation) diff --git a/src/running/command/log_preprocessor.py b/src/running/command/log_preprocessor.py index caa1ae0..1a9599e 100644 --- a/src/running/command/log_preprocessor.py +++ b/src/running/command/log_preprocessor.py @@ -8,8 +8,12 @@ from running.config import Configuration import os -MMTk_HEADER = "============================ MMTk Statistics Totals ============================" -MMTk_FOOTER = "------------------------------ End MMTk Statistics -----------------------------" +MMTk_HEADER = ( + "============================ MMTk Statistics Totals ============================" +) +MMTk_FOOTER = ( + "------------------------------ End MMTk Statistics -----------------------------" +) class EditingMode(enum.Enum): @@ -30,6 +34,7 @@ def setup_parser(subparsers): def filter_stats(predicate: Callable[[str], bool]): def inner(stats: Dict[str, float]): return {k: v for (k, v) in stats.items() if predicate(k)} + return inner @@ -43,6 +48,7 @@ def inner(stats: Dict[str, float]): new_stats = deepcopy(stats) new_stats[new_column] = functools.reduce(func, to_reduce) return new_stats + return inner @@ -64,6 +70,7 @@ def inner(stats: Dict[str, float]): new_column = k.replace(".total", ".ratio") new_stats[new_column] = v / stats[aggregated_column] return new_stats + return inner @@ -79,6 +86,7 @@ def inner(stats: Dict[str, float]): new_stats["{}.ratio".format(stw_key)] = gc / total new_stats["{}.ratio".format(other_key)] = mu / total return new_stats + return inner @@ -91,8 +99,7 @@ def calc_ipc(stats: Dict[str, float]): if cycles == 0: assert inst == 0 continue - new_stats["INSTRUCTIONS_PER_CYCLE.{}".format( - phase)] = inst / cycles + new_stats["INSTRUCTIONS_PER_CYCLE.{}".format(phase)] = inst / cycles return new_stats @@ -102,10 +109,10 @@ def calc_work_ipc(stats: Dict[str, float]): new_stats = deepcopy(stats) for (k, v) in stats.items(): if compiled.match(k): - cycles = k.replace("PERF_COUNT_HW_INSTRUCTIONS", - "PERF_COUNT_HW_CPU_CYCLES") - ipc = k.replace("PERF_COUNT_HW_INSTRUCTIONS.total", - "INSTRUCTIONS_PER_CYCLE.ratio") + cycles = k.replace("PERF_COUNT_HW_INSTRUCTIONS", "PERF_COUNT_HW_CPU_CYCLES") + ipc = k.replace( + "PERF_COUNT_HW_INSTRUCTIONS.total", "INSTRUCTIONS_PER_CYCLE.ratio" + ) new_stats[ipc] = stats[k] / stats[cycles] return new_stats @@ -140,8 +147,9 @@ def process_lines(configuration: Configuration, lines: List[str]): funcs.append(ratio_event(v)) elif f["name"] == "filter_stats": patterns_to_keep = f["val"].split(",") - funcs.append(filter_stats(lambda n: any( - [p in n for p in patterns_to_keep]))) + funcs.append( + filter_stats(lambda n: any([p in n for p in patterns_to_keep])) + ) elif f["name"] == "calc_ipc": funcs.append(calc_ipc) else: @@ -162,15 +170,13 @@ def process_lines(configuration: Configuration, lines: List[str]): elif editing == EditingMode.Values: values = map(float, line.strip().split("\t")) stats = dict(zip(names, values)) - new_stats = functools.reduce( - lambda accum, val: val(accum), funcs, stats) + new_stats = functools.reduce(lambda accum, val: val(accum), funcs, stats) if len(new_stats): new_stat_list = list(new_stats.items()) new_stat_list.sort(key=lambda x: stat_sort_helper(x[0], x[1])) new_names, new_values = list(zip(*new_stat_list)) new_lines.append("{}\n".format("\t".join(new_names))) - new_lines.append("{}\n".format( - "\t".join(map(str, new_values)))) + new_lines.append("{}\n".format("\t".join(map(str, new_values)))) else: new_lines.append("empty_after_preprocessing\n") new_lines.append("0\n") @@ -200,8 +206,7 @@ def process(configuration: Configuration, source: Path, target: Path): def run(args): if args.get("which") != "preproc": return False - configuration = Configuration.from_file( - Path(os.getcwd()), args.get("CONFIG")) + configuration = Configuration.from_file(Path(os.getcwd()), args.get("CONFIG")) source = args.get("SOURCE") target = args.get("TARGET") target.mkdir(parents=True, exist_ok=True) diff --git a/src/running/command/minheap.py b/src/running/command/minheap.py index 63bc630..e9a2153 100644 --- a/src/running/command/minheap.py +++ b/src/running/command/minheap.py @@ -30,14 +30,21 @@ class ContinueSearch(Enum): HeapTooSmall = 3 -def run_bm_with_retry(suite: BenchmarkSuite, runtime: Runtime, bm_with_heapsize: Benchmark, minheap_dir: Path, attempts: int) -> ContinueSearch: +def run_bm_with_retry( + suite: BenchmarkSuite, + runtime: Runtime, + bm_with_heapsize: Benchmark, + minheap_dir: Path, + attempts: int, +) -> ContinueSearch: def log(s): return print(s, end="", flush=True) log(" ") for _ in range(attempts): output, _companion_output, subprocess_exit = bm_with_heapsize.run( - runtime, cwd=minheap_dir) + runtime, cwd=minheap_dir + ) if runtime.is_oom(output): # if OOM is detected, we exit the loop regardless the exit statussour log("x ") @@ -59,20 +66,28 @@ def log(s): return ContinueSearch.Abort -def minheap_one_bm(suite: BenchmarkSuite, runtime: Runtime, bm: Benchmark, heap: int, minheap_dir: Path, attempts: int) -> float: +def minheap_one_bm( + suite: BenchmarkSuite, + runtime: Runtime, + bm: Benchmark, + heap: int, + minheap_dir: Path, + attempts: int, +) -> float: lo = 2 hi = heap mid = (lo + hi) // 2 - minh = float('inf') + minh = float("inf") while hi - lo > 1: heapsize = runtime.get_heapsize_modifiers(mid) size_str = "{}M".format(mid) print(size_str, end="", flush=True) bm_with_heapsize = bm.attach_modifiers(heapsize) result = run_bm_with_retry( - suite, runtime, bm_with_heapsize, minheap_dir, attempts) + suite, runtime, bm_with_heapsize, minheap_dir, attempts + ) if result is ContinueSearch.Abort: - return float('inf') + return float("inf") elif result is ContinueSearch.HeapTooBig: minh = mid hi = mid @@ -85,7 +100,12 @@ def minheap_one_bm(suite: BenchmarkSuite, runtime: Runtime, bm: Benchmark, heap: return minh -def run_with_persistence(result: Dict[str, Any], minheap_dir: Path, result_file: Optional[Path], attempts: int): +def run_with_persistence( + result: Dict[str, Any], + minheap_dir: Path, + result_file: Optional[Path], + attempts: int, +): suites = configuration.get("suites") maxheap = configuration.get("maxheap") for c in configuration.get("configs"): @@ -95,8 +115,7 @@ def run_with_persistence(result: Dict[str, Any], minheap_dir: Path, result_file: runtime, mods = parse_config_str(configuration, c) print("{} ".format(c_encoded)) if isinstance(runtime, NativeExecutable): - logging.warning( - "Minheap measurement not supported for NativeExecutable") + logging.warning("Minheap measurement not supported for NativeExecutable") continue for suite_name, bms in configuration.get("benchmarks").items(): if suite_name not in result[c_encoded]: @@ -109,9 +128,11 @@ def run_with_persistence(result: Dict[str, Any], minheap_dir: Path, result_file: print("\t {}-{} ".format(b.suite_name, b.name), end="") mod_b = b.attach_modifiers(mods) mod_b = mod_b.attach_modifiers( - b.get_runtime_specific_modifiers(runtime)) + b.get_runtime_specific_modifiers(runtime) + ) minheap = minheap_one_bm( - suite, runtime, mod_b, maxheap, minheap_dir, attempts) + suite, runtime, mod_b, maxheap, minheap_dir, attempts + ) print("minheap {}".format(minheap)) result[c_encoded][suite_name][b.name] = minheap if result_file: @@ -121,7 +142,7 @@ def run_with_persistence(result: Dict[str, Any], minheap_dir: Path, result_file: def print_best(result: Dict[str, Dict[str, Dict[str, float]]]): minheap: DefaultDict[str, DefaultDict[str, float]] - minheap = defaultdict(lambda: defaultdict(lambda: float('inf'))) + minheap = defaultdict(lambda: defaultdict(lambda: float("inf"))) minheap_config: DefaultDict[str, DefaultDict[str, str]] minheap_config = defaultdict(lambda: defaultdict(lambda: "ALL_FAILED")) for config, suites in result.items(): @@ -139,8 +160,11 @@ def print_best(result: Dict[str, Dict[str, Dict[str, float]]]): if config_best_count.items(): config, count = max(config_best_count.items(), key=lambda x: x[1]) - print("{} obtained the most number of smallest minheap sizes: {}".format( - config, count)) + print( + "{} obtained the most number of smallest minheap sizes: {}".format( + config, count + ) + ) print("Minheap configuration to be copied to runbms config files") print(yaml.dump(result[config])) @@ -149,8 +173,7 @@ def run(args): if args.get("which") != "minheap": return False global configuration - configuration = Configuration.from_file( - Path(os.getcwd()), args.get("CONFIG")) + configuration = Configuration.from_file(Path(os.getcwd()), args.get("CONFIG")) configuration.resolve_class() result_file = args.get("RESULT") if result_file.exists(): @@ -168,7 +191,6 @@ def run(args): if is_dry_run(): run_with_persistence(result, Path(minheap_dir), None, attempts) else: - run_with_persistence(result, Path( - minheap_dir), result_file, attempts) + run_with_persistence(result, Path(minheap_dir), result_file, attempts) print_best(result) return True diff --git a/src/running/command/runbms.py b/src/running/command/runbms.py index ebfa841..4ee664a 100644 --- a/src/running/command/runbms.py +++ b/src/running/command/runbms.py @@ -1,10 +1,26 @@ import logging -from typing import DefaultDict, Dict, List, Any, Optional, Set, Tuple, BinaryIO, TYPE_CHECKING +from typing import ( + DefaultDict, + Dict, + List, + Any, + Optional, + Set, + Tuple, + BinaryIO, + TYPE_CHECKING, +) from running.suite import BenchmarkSuite, is_dry_run from running.benchmark import Benchmark, SubprocessrExit from running.config import Configuration from pathlib import Path -from running.util import parse_config_str, system, get_logged_in_users, config_index_to_chr, config_str_encode +from running.util import ( + parse_config_str, + system, + get_logged_in_users, + config_index_to_chr, + config_str_encode, +) import socket from datetime import datetime from running.runtime import Runtime @@ -14,6 +30,7 @@ from running.command.fillin import fillin import math import yaml + if TYPE_CHECKING: from running.plugin.runbms import RunbmsPlugin from running.__version__ import __VERSION__ @@ -33,8 +50,8 @@ def setup_parser(subparsers): f.set_defaults(which="runbms") f.add_argument("LOG_DIR", type=Path) f.add_argument("CONFIG", type=Path) - f.add_argument("N", type=int, nargs='?') - f.add_argument("n", type=int, nargs='*') + f.add_argument("N", type=int, nargs="?") + f.add_argument("n", type=int, nargs="*") f.add_argument("-i", "--invocations", type=int) f.add_argument("-s", "--slice", type=str) f.add_argument("-p", "--id-prefix") @@ -43,8 +60,9 @@ def setup_parser(subparsers): f.add_argument("--skip-timeout", type=int) f.add_argument("--resume", type=str) f.add_argument("--workdir", type=Path) - f.add_argument("--skip-log-compression", action="store_true", - help="Skip compressing log files") + f.add_argument( + "--skip-log-compression", action="store_true", help="Skip compressing log files" + ) def getid() -> str: @@ -95,26 +113,30 @@ def spread(spread_factor: int, N: int, n: int) -> float: n: int Nominator """ - sum_1_n_minus_1 = (n*n - n) / 2 + sum_1_n_minus_1 = (n * n - n) / 2 return n + spread_factor / (N - 1) * sum_1_n_minus_1 def hfac_str(hfac: float) -> str: - return str(int(hfac*1000)) + return str(int(hfac * 1000)) def get_heapsize(hfac: float, minheap: int) -> int: return round(minheap * hfac * minheap_multiplier) -def get_hfacs(heap_range: int, spread_factor: int, N: int, ns: List[int]) -> List[float]: +def get_hfacs( + heap_range: int, spread_factor: int, N: int, ns: List[int] +) -> List[float]: start = 1.0 end = float(heap_range) - divisor = spread(spread_factor, N, N)/(end-start) - return [spread(spread_factor, N, n)/divisor + start for n in ns] + divisor = spread(spread_factor, N, N) / (end - start) + return [spread(spread_factor, N, n) / divisor + start for n in ns] -def run_benchmark_with_config(c: str, b: Benchmark, runbms_dir: Path, size: Optional[int], fd: Optional[BinaryIO]) -> Tuple[bytes, SubprocessrExit]: +def run_benchmark_with_config( + c: str, b: Benchmark, runbms_dir: Path, size: Optional[int], fd: Optional[BinaryIO] +) -> Tuple[bytes, SubprocessrExit]: runtime, mods = parse_config_str(configuration, c) mod_b = b.attach_modifiers(mods) mod_b = mod_b.attach_modifiers(b.get_runtime_specific_modifiers(runtime)) @@ -135,7 +157,9 @@ def run_benchmark_with_config(c: str, b: Benchmark, runbms_dir: Path, size: Opti return output, exit_status -def get_filename_no_ext(bm: Benchmark, hfac: Optional[float], size: Optional[int], config: str) -> str: +def get_filename_no_ext( + bm: Benchmark, hfac: Optional[float], size: Optional[int], config: str +) -> str: # If we have / in benchmark names, replace with -. safe_bm_name = bm.name.replace("/", "-") return "{}.{}.{}.{}.{}".format( @@ -149,11 +173,15 @@ def get_filename_no_ext(bm: Benchmark, hfac: Optional[float], size: Optional[int ) -def get_filename(bm: Benchmark, hfac: Optional[float], size: Optional[int], config: str) -> str: +def get_filename( + bm: Benchmark, hfac: Optional[float], size: Optional[int], config: str +) -> str: return get_filename_no_ext(bm, hfac, size, config) + ".log" -def get_filename_completed(bm: Benchmark, hfac: Optional[float], size: Optional[int], config: str) -> str: +def get_filename_completed( + bm: Benchmark, hfac: Optional[float], size: Optional[int], config: str +) -> str: return "{}.gz".format(get_filename(bm, hfac, size, config)) @@ -190,13 +218,25 @@ def get_log_prologue(runtime: Runtime, bm: Benchmark) -> str: for i in range(0, int(cores)): output += "Frequency of cpu {}: ".format(i) output += hz_to_ghz( - system("cat /sys/devices/system/cpu/cpu{}/cpufreq/scaling_cur_freq".format(i))) + system( + "cat /sys/devices/system/cpu/cpu{}/cpufreq/scaling_cur_freq".format( + i + ) + ) + ) output += "\n" output += "Governor of cpu {}: ".format(i) - output += system("cat /sys/devices/system/cpu/cpu{}/cpufreq/scaling_governor".format(i)) + output += system( + "cat /sys/devices/system/cpu/cpu{}/cpufreq/scaling_governor".format(i) + ) output += "Scaling_min_freq of cpu {}: ".format(i) output += hz_to_ghz( - system("cat /sys/devices/system/cpu/cpu{}/cpufreq/scaling_min_freq".format(i))) + system( + "cat /sys/devices/system/cpu/cpu{}/cpufreq/scaling_min_freq".format( + i + ) + ) + ) output += "\n" return output @@ -208,7 +248,7 @@ def run_one_benchmark( hfac: Optional[float], configs: List[str], runbms_dir: Path, - log_dir: Path + log_dir: Path, ): p: "RunbmsPlugin" bm_name = bm.name @@ -229,8 +269,9 @@ def run_one_benchmark( logged_in_users: Set[str] logged_in_users = get_logged_in_users() if len(logged_in_users) > 1: - logging.warning("More than one user logged in: {}".format( - " ".join(logged_in_users))) + logging.warning( + "More than one user logged in: {}".format(" ".join(logged_in_users)) + ) ever_ran = [False] * len(configs) for i in range(0, invocations): for p in plugins.values(): @@ -247,8 +288,7 @@ def run_one_benchmark( print(".", end="", flush=True) continue if resume: - log_filename_completed = get_filename_completed( - bm, hfac, size, c) + log_filename_completed = get_filename_completed(bm, hfac, size, c) if (log_dir / log_filename_completed).exists(): print(config_index_to_chr(j), end="", flush=True) continue @@ -297,10 +337,7 @@ def run_one_benchmark( # config for a particular benchmark/hfac (method parameters) if not is_dry_run() and ever_ran[j]: if not skip_log_compression: - subprocess.check_call([ - "gzip", - log_dir / log_filename - ]) + subprocess.check_call(["gzip", log_dir / log_filename]) print() @@ -311,7 +348,7 @@ def run_one_hfac( benchmarks: Dict[str, List[Benchmark]], configs: List[str], runbms_dir: Path, - log_dir: Path + log_dir: Path, ): p: "RunbmsPlugin" for p in plugins.values(): @@ -319,8 +356,9 @@ def run_one_hfac( for suite_name, bms in benchmarks.items(): suite = suites[suite_name] for bm in bms: - run_one_benchmark(invocations, suite, bm, hfac, - configs, runbms_dir, log_dir) + run_one_benchmark( + invocations, suite, bm, hfac, configs, runbms_dir, log_dir + ) rsync(log_dir) for p in plugins.values(): p.end_hfac(hfac) @@ -374,8 +412,7 @@ def run(args): skip_log_compression = args.get("skip_log_compression") # Load from configuration file global configuration - configuration = Configuration.from_file( - Path(os.getcwd()), args.get("CONFIG")) + configuration = Configuration.from_file(Path(os.getcwd()), args.get("CONFIG")) # Save metadata if not is_dry_run(): with (log_dir / "runbms.yml").open("w") as fd: @@ -407,25 +444,29 @@ def run(args): plugins = {} else: from running.plugin.runbms import RunbmsPlugin + if type(plugins) is not dict: raise TypeError("plugins must be a dictionary") - plugins = {k: RunbmsPlugin.from_config( - k, v) for k, v in plugins.items()} + plugins = {k: RunbmsPlugin.from_config(k, v) for k, v in plugins.items()} for p in plugins.values(): p.set_run_id(run_id) p.set_runbms_dir(runbms_dir) p.set_log_dir(log_dir) def run_hfacs(hfacs): - logging.info("hfacs: {}".format( - ", ".join([ - hfac_str(hfac) - for hfac in hfacs - ]) - )) + logging.info( + "hfacs: {}".format(", ".join([hfac_str(hfac) for hfac in hfacs])) + ) for hfac in hfacs: - run_one_hfac(invocations, hfac, suites, benchmarks, - configs, Path(runbms_dir), log_dir) + run_one_hfac( + invocations, + hfac, + suites, + benchmarks, + configs, + Path(runbms_dir), + log_dir, + ) print() def run_N_ns(N, ns): @@ -437,8 +478,15 @@ def run_N_ns(N, ns): return True if N is None: - run_one_hfac(invocations, None, suites, benchmarks, - configs, Path(runbms_dir), log_dir) + run_one_hfac( + invocations, + None, + suites, + benchmarks, + configs, + Path(runbms_dir), + log_dir, + ) return True if len(ns) == 0: diff --git a/src/running/config.py b/src/running/config.py index a5c77a9..caf4c5a 100644 --- a/src/running/config.py +++ b/src/running/config.py @@ -17,7 +17,7 @@ def load_class(cls, config): KEY_CLASS_MAPPING = { "suites": BenchmarkSuite, "modifiers": Modifier, - "runtimes": Runtime + "runtimes": Runtime, } @@ -36,15 +36,14 @@ def resolve_class(self): For example, self.values["suites"] is a Dict[str, Dict[str, str]], where in the inner dictionary contains the string representation of a benchmark suite. - After this function returns, self.values["suites"] becomes a + After this function returns, self.values["suites"] becomes a Dict[str, BenchmarkSuite]. Change the KEY_CLASS_MAPPING to change which classes get resolved. """ for cls_name, cls in KEY_CLASS_MAPPING.items(): if cls_name in self.__items: - self.__items[cls_name] = load_class( - cls, self.__items[cls_name]) + self.__items[cls_name] = load_class(cls, self.__items[cls_name]) if "benchmarks" in self.__items: for suite_name, bms in self.__items["benchmarks"].items(): suite = self.__items["suites"][suite_name] @@ -88,9 +87,8 @@ def combine(self, other: "Configuration") -> "Configuration": "Key `{}` has been defined in one of the " "included files, and the value of `{}`, {}, " "is not an array or a dictionary. " - "Please use overrides instead.".format( - k, k, repr(v) - )) + "Please use overrides instead.".format(k, k, repr(v)) + ) new_values[k].update(copy.deepcopy(other.__items[k])) else: new_values[k] = copy.deepcopy(other.__items[k]) @@ -104,13 +102,17 @@ def parse_file(path: Path) -> Any: return config except yaml.YAMLError as e: raise SyntaxError( - "Not able to parse the configuration file, {}".format(e)) + "Not able to parse the configuration file, {}".format(e) + ) @staticmethod def from_file(in_folder: Path, p: str) -> "Configuration": expand_p = os.path.expandvars(p) - logging.info("Loading config {}, expanding to {}, relative to {}".format( - p, expand_p, in_folder)) + logging.info( + "Loading config {}, expanding to {}, relative to {}".format( + p, expand_p, in_folder + ) + ) path = Path(expand_p) if path.is_absolute(): logging.info(" is absolute") @@ -118,24 +120,23 @@ def from_file(in_folder: Path, p: str) -> "Configuration": path = in_folder.joinpath(p) logging.info(" resolved to {}".format(path)) if not path.exists(): - raise ValueError( - "Configuration not found at path '{}'".format(path)) + raise ValueError("Configuration not found at path '{}'".format(path)) if not path.is_file(): - raise ValueError( - "Configuration at path '{}' is not a file".format(path)) + raise ValueError("Configuration at path '{}' is not a file".format(path)) with path.open("r") as fd: try: config = yaml.safe_load(fd) except yaml.YAMLError as e: raise SyntaxError( - "Not able to parse the configuration file, {}".format(e)) + "Not able to parse the configuration file, {}".format(e) + ) if config is None: raise ValueError("Parsed configuration file is None") if "includes" in config: - includes = [Configuration.from_file( - path.parent, p) for p in config["includes"]] - base = functools.reduce( - lambda left, right: left.combine(right), includes) + includes = [ + Configuration.from_file(path.parent, p) for p in config["includes"] + ] + base = functools.reduce(lambda left, right: left.combine(right), includes) if "overrides" in config: for selector, new_value in config["overrides"].items(): base.override(selector, new_value) @@ -146,6 +147,7 @@ def from_file(in_folder: Path, p: str) -> "Configuration": else: if "overrides" in config: raise KeyError( - 'You specified "overrides" but not "includes". This does not make sense.') + 'You specified "overrides" but not "includes". This does not make sense.' + ) final_config = Configuration(config) return final_config diff --git a/src/running/config/base/dacapo.yml b/src/running/config/base/dacapo.yml index ac81ca9..922afbb 100644 --- a/src/running/config/base/dacapo.yml +++ b/src/running/config/base/dacapo.yml @@ -51,184 +51,80 @@ suites: timing_iteration: 3 timeout: 120 minheap_values: {} - dacapochopin-29a657f: + dacapochopin: type: DaCapo - release: evaluation - path: /usr/share/benchmarks/dacapo/dacapo-evaluation-git-29a657f.jar - minheap: adoptopenjdk-15-G1 + release: "23.11" + path: /usr/share/benchmarks/dacapo/dacapo-23.11-chopin.jar + minheap: temurin-21-G1-n-5 minheap_values: - adoptopenjdk-15-G1: - avrora: 7 - batik: 253 - biojava: 191 - cassandra: 89 - eclipse: 437 - fop: 19 - graphchi: 255 - h2: 1154 - h2o: 73 - jme: 55 - jython: 37 - # Java version '15.0.2' is incompatable with kafka: Kafka currently requires Java 8 due to 'java.lang.NoClassDefFoundError: javax/activation/DataSource' error for more recent Java releases. - # kafka: inf - luindex: 42 - lusearch: 23 - pmd: 243 - sunflow: 33 - tradebeans: 169 - tradesoap: 133 - tomcat: 25 - xalan: 9 - zxing: 96 - temurin-17-G1: - avrora: 7 - batik: 189 + temurin-21-G1: + avrora: 5 + batik: 175 biojava: 95 - eclipse: 411 - fop: 15 - graphchi: 255 - h2: 773 + cassandra: 120 + eclipse: 133 + fop: 13 + graphchi: 173 + h2: 679 + h2o: 66 jme: 29 jython: 25 - luindex: 42 + kafka: 194 + luindex: 29 lusearch: 21 - pmd: 156 + pmd: 185 + spring: 56 sunflow: 29 - tomcat: 21 - tradebeans: 131 - tradesoap: 103 - xalan: 8 - zxing: 97 - lusearch_huge: 122 # {name: lusearch_large, bm_name: lusearch, size: large} - lusearch_large: 142 # {name: lusearch_huge, bm_name: lusearch, size: huge, timing_iteration: 2} - temurin-11-G1: + tomcat: 19 + tradebeans: 107 + tradesoap: 87 + xalan: 13 + zxing: 108 + temurin-21-G1-n-5: avrora: 5 - batik: 185 + batik: 175 biojava: 95 - cassandra: 1026 - eclipse: 423 + cassandra: 128 + eclipse: 135 fop: 13 - graphchi: 255 - h2: 771 - h2o: 62 + graphchi: 175 + h2: 681 + h2o: 71 jme: 29 - jython: 27 - luindex: 40 - lusearch: 25 - pmd: 158 - sunflow: 27 + jython: 25 + kafka: 194 + luindex: 29 + lusearch: 21 + pmd: 189 + spring: 58 + sunflow: 29 tomcat: 19 - tradebeans: 129 - tradesoap: 101 - xalan: 7 - zxing: 96 - timing_iteration: 3 - timeout: 120 - dacapochopin-f480064: - type: DaCapo - release: evaluation - path: /usr/share/benchmarks/dacapo/dacapo-evaluation-git-f480064.jar - minheap: temurin-11-G1 - minheap_values: - temurin-11-G1: + tradebeans: 135 + tradesoap: 91 + xalan: 13 + zxing: 195 + temurin-21-G1-n-10: avrora: 5 - batik: 186 - biojava: 96 - cassandra: 87 - eclipse: 423 + batik: 175 + biojava: 97 + cassandra: 130 + eclipse: 135 fop: 13 - graphchi: 255 - h2: 771 - h2o: 83 + graphchi: 175 + h2: 681 + h2o: 77 jme: 29 jython: 27 - kafka: 201 - luindex: 41 - lusearch: 26 - pmd: 43 - # spring: .inf - sunflow: 27 - tomcat: 19 - tradebeans: 129 - tradesoap: 113 - xalan: 7 - zxing: 98 - temurin-17-G1: - avrora: 7 - batik: 190 - biojava: 95 - # cassandra: .inf - eclipse: 411 - fop: 15 - graphchi: 255 - h2: 773 - # h2o: .inf - jme: 29 - jython: 25 - kafka: 197 - luindex: 42 - lusearch: 25 - pmd: 45 - # spring: .inf - sunflow: 27 - tomcat: 21 - tradebeans: 131 - tradesoap: 103 - xalan: 7 - zxing: 101 - timing_iteration: 3 - timeout: 120 - dacapochopin-6e411f33: - type: DaCapo - release: evaluation - path: /usr/share/benchmarks/dacapo/dacapo-evaluation-git-6e411f33.jar - minheap: mmtk-openjdk-11-MarkCompact - minheap_values: - mmtk-openjdk-11-MarkCompact: - avrora: 8 - batik: 426 - biojava: 197 - cassandra: 117 - eclipse: 439 - fop: 24 - graphchi: 187 - h2: 1122 - h2o: 893 - jme: 236 - jython: 48 - kafka: 233 - luindex: 25 - lusearch: 36 - pmd: 291 - spring: 110 - sunflow: 37 - tradebeans: .inf - tradesoap: .inf - tomcat: 55 - xalan: 22 - zxing: 427 - mmtk-openjdk-11-G1: - avrora: 7 - batik: 1068 - biojava: 189 - cassandra: 119 - eclipse: 427 - fop: 73 - graphchi: 258 - h2: 908 - h2o: 175 - jme: 235 - jython: 309 - kafka: 219 - luindex: 39 - lusearch: 29 - pmd: 262 - spring: 78 + kafka: 194 + luindex: 29 + lusearch: 21 + pmd: 193 + spring: 59 sunflow: 33 - tomcat: 47 - tradebeans: .inf - tradesoap: .inf - xalan: 19 - zxing: 95 - timing_iteration: 3 - timeout: 120 + tomcat: 19 + tradebeans: 135 + tradesoap: 93 + xalan: 13 + zxing: 215 + timing_iteration: 5 + timeout: 1800 diff --git a/src/running/modifier.py b/src/running/modifier.py index a570000..594620c 100644 --- a/src/running/modifier.py +++ b/src/running/modifier.py @@ -1,6 +1,7 @@ from typing import Any, Dict, List, TYPE_CHECKING from running.util import register, smart_quote, split_quoted, parse_modifier_strs import copy + if TYPE_CHECKING: from running.config import Configuration @@ -14,7 +15,10 @@ def __init__(self, value_opts=None, **kwargs): self.value_opts = value_opts if "-" in self.name: raise ValueError( - "Modifier {} has - in its name. - is reserved for value options.".format(self.name)) + "Modifier {} has - in its name. - is reserved for value options.".format( + self.name + ) + ) self.__original_kwargs = kwargs self._kwargs = copy.deepcopy(kwargs) self.excludes = kwargs.get("excludes", {}) @@ -60,7 +64,7 @@ def __init__(self, value_opts=None, **kwargs): super().__init__(value_opts, **kwargs) self.val = self._kwargs["val"].split("|") - def flatten(self, configuration: 'Configuration') -> List[Modifier]: + def flatten(self, configuration: "Configuration") -> List[Modifier]: return parse_modifier_strs(configuration, self.val) def __str__(self) -> str: @@ -109,15 +113,23 @@ def __init__(self, value_opts=None, **kwargs): super().__init__(value_opts, **kwargs) if "var" not in self._kwargs: raise ValueError( - "Please specify the name of the environment variable for modifier {}".format(self.name)) + "Please specify the name of the environment variable for modifier {}".format( + self.name + ) + ) if "val" not in self._kwargs: raise ValueError( - "Please specify the value for the environment variable for modifier {}".format(self.name)) + "Please specify the value for the environment variable for modifier {}".format( + self.name + ) + ) self.var = self._kwargs["var"] self.val = self._kwargs["val"] def __str__(self) -> str: - return "{} EnvVar {}={}".format(super().__str__(), self.var, smart_quote(self.val)) + return "{} EnvVar {}={}".format( + super().__str__(), self.var, smart_quote(self.val) + ) @register(Modifier) diff --git a/src/running/plugin/runbms/__init__.py b/src/running/plugin/runbms/__init__.py index 8a21584..9803fec 100644 --- a/src/running/plugin/runbms/__init__.py +++ b/src/running/plugin/runbms/__init__.py @@ -1,5 +1,6 @@ from pathlib import Path from typing import Any, Dict, Optional, TYPE_CHECKING + if TYPE_CHECKING: from running.benchmark import Benchmark @@ -38,28 +39,62 @@ def start_hfac(self, _hfac: Optional[float]): def end_hfac(self, _hfac: Optional[float]): pass - def start_benchmark(self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchmark"): + def start_benchmark( + self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchmark" + ): pass - def end_benchmark(self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchmark"): + def end_benchmark( + self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchmark" + ): pass - def start_invocation(self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchmark", _invocation: int): + def start_invocation( + self, + _hfac: Optional[float], + _size: Optional[int], + _bm: "Benchmark", + _invocation: int, + ): pass - def end_invocation(self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchmark", _invocation: int): + def end_invocation( + self, + _hfac: Optional[float], + _size: Optional[int], + _bm: "Benchmark", + _invocation: int, + ): pass - def start_config(self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchmark", _invocation: int, _config: str, _config_index: int): + def start_config( + self, + _hfac: Optional[float], + _size: Optional[int], + _bm: "Benchmark", + _invocation: int, + _config: str, + _config_index: int, + ): pass - def end_config(self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchmark", _invocation: int, _config: str, _config_index: int, _passed: bool): + def end_config( + self, + _hfac: Optional[float], + _size: Optional[int], + _bm: "Benchmark", + _invocation: int, + _config: str, + _config_index: int, + _passed: bool, + ): pass # !!! Do NOT remove this import nor change its position # This is to make sure that the plugin classes are correctly registered from running.plugin.runbms.copyfile import CopyFile + if TYPE_CHECKING: from running.plugin.runbms.zulip import Zulip else: @@ -71,8 +106,10 @@ def end_config(self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchma @register(RunbmsPlugin) class Zulip(RunbmsPlugin): def __init__(self, **kwargs): - raise RuntimeError("Trying to create an instance of the Zulip " - "plugin for runbms, but the import failed. " - "This is most likely due to the required " - "dependencies not being installed. Try pip " - "install running-ng[zulip] to install the extra dependencies.") + raise RuntimeError( + "Trying to create an instance of the Zulip " + "plugin for runbms, but the import failed. " + "This is most likely due to the required " + "dependencies not being installed. Try pip " + "install running-ng[zulip] to install the extra dependencies." + ) diff --git a/src/running/plugin/runbms/copyfile.py b/src/running/plugin/runbms/copyfile.py index ed829e2..651e095 100644 --- a/src/running/plugin/runbms/copyfile.py +++ b/src/running/plugin/runbms/copyfile.py @@ -6,6 +6,7 @@ from running.suite import is_dry_run from running.util import register import shutil + if TYPE_CHECKING: from running.benchmark import Benchmark @@ -40,27 +41,60 @@ def end_hfac(self, _hfac: Optional[float]): if self.nop: return - def start_benchmark(self, _hfac: Optional[float], _size: Optional[int], bm: "Benchmark"): + def start_benchmark( + self, _hfac: Optional[float], _size: Optional[int], bm: "Benchmark" + ): if self.nop: return - def end_benchmark(self, _hfac: Optional[float], _size: Optional[int], bm: "Benchmark"): + def end_benchmark( + self, _hfac: Optional[float], _size: Optional[int], bm: "Benchmark" + ): if self.nop: return - def start_invocation(self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchmark", invocation: int): + def start_invocation( + self, + _hfac: Optional[float], + _size: Optional[int], + _bm: "Benchmark", + invocation: int, + ): if self.nop: return - def end_invocation(self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchmark", _invocation: int): + def end_invocation( + self, + _hfac: Optional[float], + _size: Optional[int], + _bm: "Benchmark", + _invocation: int, + ): if self.nop: return - def start_config(self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchmark", _invocation: int, config: str, _config_index: int): + def start_config( + self, + _hfac: Optional[float], + _size: Optional[int], + _bm: "Benchmark", + _invocation: int, + config: str, + _config_index: int, + ): if self.nop: return - def end_config(self, hfac: Optional[float], size: Optional[int], bm: "Benchmark", invocation: int, config: str, _config_index: int, passed: bool): + def end_config( + self, + hfac: Optional[float], + size: Optional[int], + bm: "Benchmark", + invocation: int, + config: str, + _config_index: int, + passed: bool, + ): if self.nop: return if self.runbms_dir is None: @@ -68,8 +102,7 @@ def end_config(self, hfac: Optional[float], size: Optional[int], bm: "Benchmark" if self.log_dir is None: raise ValueError("log_dir should be set") folder_name = "{}.{}".format( - get_filename_no_ext(bm, hfac, size, config), - invocation + get_filename_no_ext(bm, hfac, size, config), invocation ) if self.skip_failed and (not passed): # Do nothing if we skip failed invocation and the current invocation diff --git a/src/running/plugin/runbms/zulip.py b/src/running/plugin/runbms/zulip.py index 56a1d1e..f4ef952 100644 --- a/src/running/plugin/runbms/zulip.py +++ b/src/running/plugin/runbms/zulip.py @@ -7,12 +7,12 @@ from running.suite import is_dry_run from running.command.runbms import hfac_str from datetime import datetime, timedelta + if TYPE_CHECKING: from running.benchmark import Benchmark RESERVATION_WARNING_HOURS = 12 -RESERVATION_WARNING_THRESHOLD = timedelta( - seconds=RESERVATION_WARNING_HOURS * 60 * 60) +RESERVATION_WARNING_THRESHOLD = timedelta(seconds=RESERVATION_WARNING_HOURS * 60 * 60) @register(RunbmsPlugin) @@ -38,9 +38,7 @@ def __init__(self, **kwargs): def send_message(self, content): message_data = copy.deepcopy(self.request) message_data["content"] = "{}\n{}{}\n".format( - self.run_id, - self.get_reservation_message(), - content + self.run_id, self.get_reservation_message(), content ) try: result = self.client.send_message(message_data=message_data) @@ -60,8 +58,7 @@ def modify_message(self, content): try: result = self.client.update_message(request) if result["result"] != "success": - logging.warning( - "Zulip update_message failed\n{}".format(result)) + logging.warning("Zulip update_message failed\n{}".format(result)) else: self.last_message_content = content except: @@ -73,46 +70,82 @@ def __str__(self) -> str: def start_hfac(self, hfac: Optional[float]): if self.nop: return - self.send_message("hfac {} started".format( - hfac_str(hfac) if hfac is not None else "None")) + self.send_message( + "hfac {} started".format(hfac_str(hfac) if hfac is not None else "None") + ) def end_hfac(self, hfac: Optional[float]): if self.nop: return - self.send_message("hfac {} ended".format( - hfac_str(hfac) if hfac is not None else "None")) + self.send_message( + "hfac {} ended".format(hfac_str(hfac) if hfac is not None else "None") + ) - def start_benchmark(self, _hfac: Optional[float], _size: Optional[int], bm: "Benchmark"): + def start_benchmark( + self, _hfac: Optional[float], _size: Optional[int], bm: "Benchmark" + ): if self.nop: return self.send_message("benchmark {} started".format(bm.name)) - def end_benchmark(self, _hfac: Optional[float], _size: Optional[int], bm: "Benchmark"): + def end_benchmark( + self, _hfac: Optional[float], _size: Optional[int], bm: "Benchmark" + ): if self.nop: return self.send_message("benchmark {} ended".format(bm.name)) - def start_invocation(self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchmark", invocation: int): + def start_invocation( + self, + _hfac: Optional[float], + _size: Optional[int], + _bm: "Benchmark", + invocation: int, + ): if self.nop: return if self.last_message_id and self.last_message_content: self.modify_message(self.last_message_content + str(invocation)) - def end_invocation(self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchmark", _invocation: int): + def end_invocation( + self, + _hfac: Optional[float], + _size: Optional[int], + _bm: "Benchmark", + _invocation: int, + ): if self.nop: return - def start_config(self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchmark", _invocation: int, _config: str, _config_index: int): + def start_config( + self, + _hfac: Optional[float], + _size: Optional[int], + _bm: "Benchmark", + _invocation: int, + _config: str, + _config_index: int, + ): if self.nop: return - def end_config(self, _hfac: Optional[float], _size: Optional[int], _bm: "Benchmark", _invocation: int, _config: str, config_index: int, passed: bool): + def end_config( + self, + _hfac: Optional[float], + _size: Optional[int], + _bm: "Benchmark", + _invocation: int, + _config: str, + config_index: int, + passed: bool, + ): if self.nop: return if self.last_message_id and self.last_message_content: if passed: - self.modify_message(self.last_message_content + - config_index_to_chr(config_index)) + self.modify_message( + self.last_message_content + config_index_to_chr(config_index) + ) else: self.modify_message(self.last_message_content + ".") @@ -126,16 +159,14 @@ def get_reservation_message(self) -> str: return "# ** Warning: machine not reserved. **\n" elif reservation.status is MomaReservationStatus.RESERVED_BY_OTHERS: return "# ** Warning: machine reserved by {}, ends at {}. **\n".format( - reservation.user, - reservation.end + reservation.user, reservation.end ) elif reservation.status is MomaReservationStatus.RESERVED_BY_ME: assert reservation.end is not None delta = reservation.end - datetime.now() if delta < RESERVATION_WARNING_THRESHOLD: return "# ** Warning: less than {} hours of reservation left. Current reservation ends at {}. **\n".format( - RESERVATION_WARNING_HOURS, - reservation.end + RESERVATION_WARNING_HOURS, reservation.end ) else: return "" diff --git a/src/running/runtime.py b/src/running/runtime.py index b671861..e6f0286 100644 --- a/src/running/runtime.py +++ b/src/running/runtime.py @@ -64,12 +64,17 @@ def get_heapsize_modifiers(self, size: int) -> List[Modifier]: size_str = "{}M".format(size) heapsize = JVMArg( name="heap{}".format(size_str), - val="-Xms{} -Xmx{}".format(size_str, size_str) + val="-Xms{} -Xmx{}".format(size_str, size_str), ) return [heapsize] def is_oom(self, output: bytes) -> bool: - for pattern in [b"Allocation Failed", b"OutOfMemoryError", b"ran out of memory", b"panicked at 'Out of memory!'"]: + for pattern in [ + b"Allocation Failed", + b"OutOfMemoryError", + b"ran out of memory", + b"panicked at 'Out of memory!'", + ]: if pattern in output: return True return False @@ -90,8 +95,7 @@ def __init__(self, **kwargs): logging.warning("OpenJDK home {} doesn't exist".format(self.home)) self.executable = self.home / "bin" / "java" if not self.executable.exists(): - logging.warning( - "{} not found in OpenJDK home".format(self.executable)) + logging.warning("{} not found in OpenJDK home".format(self.executable)) self.executable = self.executable.absolute() def get_executable(self) -> Path: @@ -111,8 +115,7 @@ def __init__(self, **kwargs): logging.warning("JikesRVM home {} doesn't exist".format(self.home)) self.executable = self.home / "rvm" if not self.home.exists(): - logging.warning( - "{} not found in JikesRVM home".format(self.executable)) + logging.warning("{} not found in JikesRVM home".format(self.executable)) self.executable = self.executable.absolute() def get_executable(self) -> Path: @@ -129,7 +132,8 @@ def __init__(self, **kwargs): self.executable = Path(kwargs["executable"]) if not self.executable.exists(): logging.warning( - "JavaScriptRuntime executable {} doesn't exist".format(self.executable)) + "JavaScriptRuntime executable {} doesn't exist".format(self.executable) + ) self.executable = self.executable.absolute() def get_executable(self) -> Path: @@ -145,8 +149,7 @@ def get_heapsize_modifiers(self, size: int) -> List[Modifier]: size_str = "{}".format(size) heapsize = JSArg( name="heap{}".format(size_str), - val="--initial-heap-size={} --max-heap-size={}".format( - size_str, size_str) + val="--initial-heap-size={} --max-heap-size={}".format(size_str, size_str), ) return [heapsize] @@ -170,8 +173,7 @@ def get_heapsize_modifiers(self, size: int) -> List[Modifier]: size_str = "{}".format(size) # FIXME doesn't seem to be working heapsize = JSArg( - name="heap{}".format(size_str), - val="--available-memory={}".format(size_str) + name="heap{}".format(size_str), val="--available-memory={}".format(size_str) ) return [heapsize] @@ -189,8 +191,7 @@ def get_heapsize_modifiers(self, size: int) -> List[Modifier]: size_str = "{}".format(size) # FIXME doesn't seem to be working heapsize = JSArg( - name="heap{}".format(size_str), - val="--gcMaxHeapSize={}".format(size_str) + name="heap{}".format(size_str), val="--gcMaxHeapSize={}".format(size_str) ) return [heapsize] @@ -205,8 +206,7 @@ def __init__(self, **kwargs): self.executable: Path self.executable = Path(kwargs["executable"]) if not self.executable.exists(): - logging.warning( - "Julia executable {} doesn't exist".format(self.executable)) + logging.warning("Julia executable {} doesn't exist".format(self.executable)) self.executable = self.executable.absolute() def get_executable(self) -> Path: @@ -221,10 +221,12 @@ class JuliaMMTK(Julia): def get_heapsize_modifiers(self, size: int) -> List[Modifier]: # size in MB size_str = "{}".format(size) - min = EnvVar(name="minheap{}".format(size_str), - var="MMTK_MIN_HSIZE", val=size_str) - max = EnvVar(name="maxheap{}".format(size_str), - var="MMTK_MAX_HSIZE", val=size_str) + min = EnvVar( + name="minheap{}".format(size_str), var="MMTK_MIN_HSIZE", val=size_str + ) + max = EnvVar( + name="maxheap{}".format(size_str), var="MMTK_MAX_HSIZE", val=size_str + ) return [min, max] def __str__(self): diff --git a/src/running/suite.py b/src/running/suite.py index a19a5cf..c97d33a 100644 --- a/src/running/suite.py +++ b/src/running/suite.py @@ -1,6 +1,12 @@ from pathlib import Path from typing import Any, Dict, Optional, Union, List, Sequence -from running.benchmark import JavaBenchmark, BinaryBenchmark, Benchmark, JavaScriptBenchmark, JuliaBenchmark +from running.benchmark import ( + JavaBenchmark, + BinaryBenchmark, + Benchmark, + JavaScriptBenchmark, + JuliaBenchmark, +) from running.runtime import OpenJDK, Runtime from running.modifier import JVMArg, Modifier import logging @@ -23,7 +29,8 @@ def set_dry_run(val: bool): def parse_timing_iteration(t: Optional[str], suite_name: str) -> Union[str, int]: if not t: raise KeyError( - "You need to specify the timing_iteration for a {} suite".format(suite_name)) + "You need to specify the timing_iteration for a {} suite".format(suite_name) + ) assert t is not None try: t_parsed = int(t) @@ -62,23 +69,20 @@ def __init__(self, programs: Dict[str, Dict[str, str]], **kwargs): super().__init__(**kwargs) self.programs: Dict[str, Dict[str, Any]] self.programs = { - k: { - 'path': Path(v['path']), - 'args': split_quoted(v['args']) - } + k: {"path": Path(v["path"]), "args": split_quoted(v["args"])} for k, v in programs.items() } self.timeout = kwargs.get("timeout") - def get_benchmark(self, bm_spec: Union[str, Dict[str, Any]]) -> 'BinaryBenchmark': + def get_benchmark(self, bm_spec: Union[str, Dict[str, Any]]) -> "BinaryBenchmark": assert type(bm_spec) is str bm_name = bm_spec return BinaryBenchmark( - self.programs[bm_name]['path'], - self.programs[bm_name]['args'], + self.programs[bm_name]["path"], + self.programs[bm_name]["args"], suite_name=self.name, name=bm_name, - timeout=self.timeout + timeout=self.timeout, ) def get_minheap(self, bm: Benchmark) -> int: @@ -106,8 +110,7 @@ def __init__(self, **kwargs): self.release: str self.release = kwargs["release"] if self.release not in ["2006", "9.12", "evaluation"]: - raise ValueError( - "DaCapo release {} not recongized".format(self.release)) + raise ValueError("DaCapo release {} not recongized".format(self.release)) self.path: Path self.path = Path(kwargs["path"]) if not self.path.exists(): @@ -118,21 +121,33 @@ def __init__(self, **kwargs): self.minheap_values = kwargs.get("minheap_values", {}) if not isinstance(self.minheap_values, dict): raise TypeError( - "The minheap_values of {} should be a dictionary".format(self.name)) + "The minheap_values of {} should be a dictionary".format(self.name) + ) if self.minheap: if not isinstance(self.minheap, str): raise TypeError( - "The minheap of {} should be a string that selects from a minheap_values".format(self.name)) + "The minheap of {} should be a string that selects from a minheap_values".format( + self.name + ) + ) if self.minheap not in self.minheap_values: raise KeyError( - "{} is not a valid entry of {}.minheap_values".format(self.name, self.name)) + "{} is not a valid entry of {}.minheap_values".format( + self.name, self.name + ) + ) self.timing_iteration = parse_timing_iteration( - kwargs.get("timing_iteration"), "DaCapo") - if isinstance(self.timing_iteration, str) and self.timing_iteration != "converge": - raise TypeError("The timing iteration of the DaCapo benchmark suite `{}` is {}, which neither an integer nor 'converge'".format( - self.path, - repr(self.timing_iteration) - )) + kwargs.get("timing_iteration"), "DaCapo" + ) + if ( + isinstance(self.timing_iteration, str) + and self.timing_iteration != "converge" + ): + raise TypeError( + "The timing iteration of the DaCapo benchmark suite `{}` is {}, which neither an integer nor 'converge'".format( + self.path, repr(self.timing_iteration) + ) + ) self.callback: Optional[str] self.callback = kwargs.get("callback") self.timeout: Optional[int] @@ -154,13 +169,15 @@ def parse_timing_iteration(v: Any): timing_iteration = int(v) except ValueError: if v != "converge": - raise TypeError("The timing iteration {} is neither an integer nor 'converge'".format( - repr(v) - )) + raise TypeError( + "The timing iteration {} is neither an integer nor 'converge'".format( + repr(v) + ) + ) timing_iteration = v return timing_iteration - def get_benchmark(self, bm_spec: Union[str, Dict[str, Any]]) -> 'JavaBenchmark': + def get_benchmark(self, bm_spec: Union[str, Dict[str, Any]]) -> "JavaBenchmark": timing_iteration = self.timing_iteration timeout = self.timeout size = self.size @@ -171,12 +188,14 @@ def get_benchmark(self, bm_spec: Union[str, Dict[str, Any]]) -> 'JavaBenchmark': assert type(bm_spec) is dict if "bm_name" not in bm_spec or "name" not in bm_spec: raise KeyError( - "When a dictionary is used to speicfy a benchmark, you need to provide both `name` and `bm_name`") + "When a dictionary is used to speicfy a benchmark, you need to provide both `name` and `bm_name`" + ) bm_name = bm_spec["bm_name"] name = bm_spec["name"] if "timing_iteration" in bm_spec: timing_iteration = DaCapo.parse_timing_iteration( - bm_spec["timing_iteration"]) + bm_spec["timing_iteration"] + ) # user overriding the size for that benchmark if "size" in bm_spec: size = bm_spec["size"] @@ -210,10 +229,12 @@ def strategy(runtime: Runtime) -> Sequence[Modifier]: modifiers = [] if isinstance(runtime, OpenJDK): if runtime.release >= 9: - modifiers.append(JVMArg( - name="add_exports", - val="--add-exports java.base/jdk.internal.ref=ALL-UNNAMED" - )) + modifiers.append( + JVMArg( + name="add_exports", + val="--add-exports java.base/jdk.internal.ref=ALL-UNNAMED", + ) + ) return modifiers return JavaBenchmark( @@ -225,20 +246,18 @@ def strategy(runtime: Runtime) -> Sequence[Modifier]: suite_name=self.name, name=name, timeout=timeout, - runtime_specific_modifiers_strategy=strategy + runtime_specific_modifiers_strategy=strategy, ) def get_minheap(self, bm: Benchmark) -> int: assert isinstance(bm, JavaBenchmark) name = bm.name if not self.minheap: - logging.warning( - "No minheap_value of {} is selected".format(self)) + logging.warning("No minheap_value of {} is selected".format(self)) return __DEFAULT_MINHEAP minheap = self.minheap_values[self.minheap] if name not in minheap: - logging.warning( - "Minheap for {} of {} not set".format(name, self)) + logging.warning("Minheap for {} of {} not set".format(name, self)) return __DEFAULT_MINHEAP return minheap[name] @@ -253,10 +272,12 @@ def get_wrapper(self, bm_name: str) -> Optional[str]: elif type(self.wrapper) == dict: return self.wrapper.get(bm_name) else: - raise TypeError("wrapper of {} must be either null, " - "a string (the same wrapper for all benchmarks), " - "or a dictionary (different wrappers for" - "differerent benchmarks)".format(self.name)) + raise TypeError( + "wrapper of {} must be either null, " + "a string (the same wrapper for all benchmarks), " + "or a dictionary (different wrappers for" + "differerent benchmarks)".format(self.name) + ) def get_companion(self, bm_name: str) -> Optional[str]: if self.companion is None: @@ -266,10 +287,12 @@ def get_companion(self, bm_name: str) -> Optional[str]: elif type(self.companion) == dict: return self.companion.get(bm_name) else: - raise TypeError("companion of {} must be either null, " - "a string (the same companion for all benchmarks), " - "or a dictionary (different companions for" - "differerent benchmarks)".format(self.name)) + raise TypeError( + "companion of {} must be either null, " + "a string (the same companion for all benchmarks), " + "or a dictionary (different companions for" + "differerent benchmarks)".format(self.name) + ) @register(BenchmarkSuite) @@ -280,34 +303,37 @@ def __init__(self, **kwargs): self.release = kwargs["release"] if self.release not in ["1.03"]: raise ValueError( - "SPECjbb2015 release {} not recongized".format(self.release)) + "SPECjbb2015 release {} not recongized".format(self.release) + ) self.path: Path self.path = Path(kwargs["path"]).resolve() - self.propsfile = (self.path / ".." / "config" / - "specjbb2015.props").resolve() + self.propsfile = (self.path / ".." / "config" / "specjbb2015.props").resolve() if not self.path.exists(): logging.info("SPECjbb2015 jar {} not found".format(self.path)) def __str__(self) -> str: return "{} SPECjbb2015 {} {}".format(super().__str__(), self.release, self.path) - def get_benchmark(self, bm_spec: Union[str, Dict[str, Any]]) -> 'JavaBenchmark': + def get_benchmark(self, bm_spec: Union[str, Dict[str, Any]]) -> "JavaBenchmark": assert type(bm_spec) is str if bm_spec != "composite": raise ValueError("Only composite mode is supported for now") program_args = [ - "-jar", str(self.path), - "-p", str(self.propsfile), - "-m", "COMPOSITE", - "-skipReport" + "-jar", + str(self.path), + "-p", + str(self.propsfile), + "-m", + "COMPOSITE", + "-skipReport", ] return JavaBenchmark( jvm_args=[], program_args=program_args, cp=[], suite_name=self.name, - name="composite" + name="composite", ) def get_minheap(self, _bm: Benchmark) -> int: @@ -331,11 +357,11 @@ def __init__(self, **kwargs): if not self.wrapper.exists(): logging.info("Octane folder {} not found".format(self.wrapper)) timing_iteration = parse_timing_iteration( - kwargs.get("timing_iteration"), "Octane") + kwargs.get("timing_iteration"), "Octane" + ) self.timing_iteration: int if isinstance(timing_iteration, str): - raise TypeError( - "timing_iteration for Octane has to be an integer") + raise TypeError("timing_iteration for Octane has to be an integer") else: self.timing_iteration = timing_iteration self.minheap: Optional[str] @@ -344,48 +370,51 @@ def __init__(self, **kwargs): self.minheap_values = kwargs.get("minheap_values", {}) if not isinstance(self.minheap_values, dict): raise TypeError( - "The minheap_values of {} should be a dictionary".format(self.name)) + "The minheap_values of {} should be a dictionary".format(self.name) + ) if self.minheap: if not isinstance(self.minheap, str): raise TypeError( - "The minheap of {} should be a string that selects from a minheap_values".format(self.name)) + "The minheap of {} should be a string that selects from a minheap_values".format( + self.name + ) + ) if self.minheap not in self.minheap_values: raise KeyError( - "{} is not a valid entry of {}.minheap_values".format(self.name, self.name)) + "{} is not a valid entry of {}.minheap_values".format( + self.name, self.name + ) + ) self.timeout: Optional[int] self.timeout = kwargs.get("timeout") def __str__(self) -> str: return "{} Octane {}".format(super().__str__(), self.path) - def get_benchmark(self, bm_spec: Union[str, Dict[str, Any]]) -> 'JavaScriptBenchmark': + def get_benchmark( + self, bm_spec: Union[str, Dict[str, Any]] + ) -> "JavaScriptBenchmark": assert type(bm_spec) is str - program_args = [ - str(self.path), - bm_spec, - str(self.timing_iteration) - ] + program_args = [str(self.path), bm_spec, str(self.timing_iteration)] return JavaScriptBenchmark( js_args=[], program=str(self.wrapper), program_args=program_args, suite_name=self.name, name=bm_spec, - timeout=self.timeout + timeout=self.timeout, ) def get_minheap(self, bm: Benchmark) -> int: assert isinstance(bm, JavaScriptBenchmark) name = bm.name if not self.minheap: - logging.warning( - "No minheap_value of {} is selected".format(self)) + logging.warning("No minheap_value of {} is selected".format(self)) return __DEFAULT_MINHEAP minheap = self.minheap_values[self.minheap] if name not in minheap: - logging.warning( - "Minheap for {} of {} not set".format(name, self)) + logging.warning("Minheap for {} of {} not set".format(name, self)) return __DEFAULT_MINHEAP return minheap[name] @@ -400,8 +429,7 @@ def __init__(self, **kwargs): self.release: str self.release = kwargs["release"] if self.release not in ["1.03_05"]: - raise ValueError( - "SPECjvm98 release {} not recongized".format(self.release)) + raise ValueError("SPECjvm98 release {} not recongized".format(self.release)) self.path: Path self.path = Path(kwargs["path"]).resolve() @@ -409,25 +437,26 @@ def __init__(self, **kwargs): logging.info("SPECjvm98 {} not found".format(self.path)) if not (self.path / "SpecApplication.class").exists(): logging.info( - "SpecApplication.class not found under SPECjvm98 {}".format(self.path)) + "SpecApplication.class not found under SPECjvm98 {}".format(self.path) + ) timing_iteration = parse_timing_iteration( - kwargs.get("timing_iteration"), "SPECjvm98") + kwargs.get("timing_iteration"), "SPECjvm98" + ) self.timing_iteration: int if isinstance(timing_iteration, str): - raise TypeError( - "timing_iteration for SPECjvm98 has to be an integer") + raise TypeError("timing_iteration for SPECjvm98 has to be an integer") else: self.timing_iteration = timing_iteration def __str__(self) -> str: return "{} SPECjvm98 {} {}".format(super().__str__(), self.release, self.path) - def get_benchmark(self, bm_spec: Union[str, Dict[str, Any]]) -> 'JavaBenchmark': + def get_benchmark(self, bm_spec: Union[str, Dict[str, Any]]) -> "JavaBenchmark": assert type(bm_spec) is str program_args = [ "SpecApplication", "-i{}".format(self.timing_iteration), - bm_spec + bm_spec, ] return JavaBenchmark( jvm_args=[], @@ -435,7 +464,7 @@ def get_benchmark(self, bm_spec: Union[str, Dict[str, Any]]) -> 'JavaBenchmark': cp=[str(self.path)], suite_name=self.name, name=bm_spec, - override_cwd=self.path + override_cwd=self.path, ) def get_minheap(self, _bm: Benchmark) -> int: @@ -454,22 +483,28 @@ def __init__(self, **kwargs): self.path: Path self.path = Path(kwargs["path"]) if not self.path.exists(): - logging.warning( - "JuliaGCBenchmarks does not exist at {}".format(self.path)) + logging.warning("JuliaGCBenchmarks does not exist at {}".format(self.path)) self.minheap: Optional[str] self.minheap = kwargs.get("minheap") self.minheap_values: Dict[str, Dict[str, int]] self.minheap_values = kwargs.get("minheap_values", {}) if not isinstance(self.minheap_values, dict): raise TypeError( - "The minheap_values of {} should be a dictionary".format(self.name)) + "The minheap_values of {} should be a dictionary".format(self.name) + ) if self.minheap: if not isinstance(self.minheap, str): raise TypeError( - "The minheap of {} should be a string that selects from a minheap_values".format(self.name)) + "The minheap of {} should be a string that selects from a minheap_values".format( + self.name + ) + ) if self.minheap not in self.minheap_values: raise KeyError( - "{} is not a valid entry of {}.minheap_values".format(self.name, self.name)) + "{} is not a valid entry of {}.minheap_values".format( + self.name, self.name + ) + ) def __str__(self) -> str: return "{} JuliaGCBenchmarks {}".format(super().__str__(), self.path) @@ -477,17 +512,15 @@ def __str__(self) -> str: def get_minheap(self, bm: Benchmark) -> int: name = bm.name if not self.minheap: - logging.warning( - "No minheap_value of {} is selected".format(self)) + logging.warning("No minheap_value of {} is selected".format(self)) return __DEFAULT_MINHEAP minheap = self.minheap_values[self.minheap] if name not in minheap: - logging.warning( - "Minheap for {} of {} not set".format(name, self)) + logging.warning("Minheap for {} of {} not set".format(name, self)) return __DEFAULT_MINHEAP return minheap[name] - def get_benchmark(self, bm_spec: Union[str, Dict[str, Any]]) -> 'JuliaBenchmark': + def get_benchmark(self, bm_spec: Union[str, Dict[str, Any]]) -> "JuliaBenchmark": assert type(bm_spec) is str return JuliaBenchmark( julia_args=[], diff --git a/src/running/util.py b/src/running/util.py index 96b2151..9486982 100644 --- a/src/running/util.py +++ b/src/running/util.py @@ -1,4 +1,5 @@ from typing import Any, List, TYPE_CHECKING, Optional, Tuple, Set + if TYPE_CHECKING: from running.config import Configuration from running.modifier import Modifier @@ -14,13 +15,16 @@ def system(cmd, check=True) -> str: - return subprocess.run(cmd, check=check, stdout=subprocess.PIPE, shell=True).stdout.decode("utf-8") + return subprocess.run( + cmd, check=check, stdout=subprocess.PIPE, shell=True + ).stdout.decode("utf-8") def register(parent_class): def inner(cls): parent_class.CLS_MAPPING[cls.__name__] = cls return cls + return inner @@ -28,14 +32,17 @@ def config_index_to_chr(i: int) -> str: if i < 0 or i >= 52: raise ValueError("Cannot convert {} into a character".format(i)) elif i < 26: - return chr(ord('a')+i) + return chr(ord("a") + i) else: - return chr(ord('A')+i-26) + return chr(ord("A") + i - 26) -def parse_modifier_strs(configuration: 'Configuration', mod_strs: List[str]) -> List['Modifier']: +def parse_modifier_strs( + configuration: "Configuration", mod_strs: List[str] +) -> List["Modifier"]: # Some modifiers could be a modifier set, and we need to flatten it from running.modifier import ModifierSet + mods = [] for m in mod_strs: m = m.strip() @@ -55,9 +62,11 @@ def parse_modifier_strs(configuration: 'Configuration', mod_strs: List[str]) -> return mods -def parse_config_str(configuration: 'Configuration', c: str) -> Tuple['Runtime', List['Modifier']]: - runtime = configuration.get("runtimes")[c.split('|')[0].strip()] - mods = parse_modifier_strs(configuration, c.split('|')[1:]) +def parse_config_str( + configuration: "Configuration", c: str +) -> Tuple["Runtime", List["Modifier"]]: + runtime = configuration.get("runtimes")[c.split("|")[0].strip()] + mods = parse_modifier_strs(configuration, c.split("|")[1:]) return runtime, mods @@ -72,14 +81,14 @@ def split_quoted(s: str) -> List[str]: def smart_quote(_s: Any) -> str: s = str(_s) if s == "": - return "\"\"" + return '""' need_quote = False for c in s: - if not (c.isalnum() or c in '.:/+=-_'): + if not (c.isalnum() or c in ".:/+=-_"): need_quote = True break if need_quote: - return "\"{}\"".format(s) + return '"{}"'.format(s) else: return s @@ -97,7 +106,13 @@ class MomaReservationStatus(enum.Enum): class MomaReservaton(object): - def __init__(self, status: MomaReservationStatus, user: Optional[str], start: Optional[datetime], end: Optional[datetime]): + def __init__( + self, + status: MomaReservationStatus, + user: Optional[str], + start: Optional[datetime], + end: Optional[datetime], + ): self.status = status self.user = user self.start = start @@ -112,12 +127,10 @@ def __init__(self, host: Optional[str] = None, frequency: int = 60): else: self.host = host try: - self.is_moma = socket.gethostbyname_ex( - self.host)[0].endswith(".moma") + self.is_moma = socket.gethostbyname_ex(self.host)[0].endswith(".moma") except socket.gaierror: self.is_moma = False - self.reserve_time_url = "http://10.0.0.1/reserve-time?host={}".format( - self.host) + self.reserve_time_url = "http://10.0.0.1/reserve-time?host={}".format(self.host) self.frequency = frequency self.last_checked: Optional[float] self.last_checked = None @@ -132,22 +145,23 @@ def update_reservation(self): return if not self.is_moma: self.reservation = MomaReservaton( - MomaReservationStatus.NOT_MOMA, - None, None, None + MomaReservationStatus.NOT_MOMA, None, None, None ) else: with urllib.request.urlopen(self.reserve_time_url) as response: html = response.read() if not html: self.reservation = MomaReservaton( - MomaReservationStatus.NOT_RESERVED, - None, None, None + MomaReservationStatus.NOT_RESERVED, None, None, None ) else: html = html.decode("utf-8") user, start, end = html.split(",") - status = MomaReservationStatus.RESERVED_BY_ME if user == getpass.getuser( - ) else MomaReservationStatus.RESERVED_BY_OTHERS + status = ( + MomaReservationStatus.RESERVED_BY_ME + if user == getpass.getuser() + else MomaReservationStatus.RESERVED_BY_OTHERS + ) start = datetime.fromtimestamp(int(start)) end = datetime.fromtimestamp(int(end)) self.reservation = MomaReservaton(status, user, start, end) diff --git a/tests/test_config.py b/tests/test_config.py index deb273d..4a9574c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -3,10 +3,7 @@ def test_override(): - c = Configuration({ - "a": {"b": 1, "c": 42}, - "d": ["foo", "bar"] - }) + c = Configuration({"a": {"b": 1, "c": 42}, "d": ["foo", "bar"]}) c.override("a.c", 43) c.override("d.1", "buzz") assert c.get("a")["b"] == 1 @@ -15,16 +12,9 @@ def test_override(): def test_combine(): - c1 = Configuration({ - "a": {"b": 1, "c": 42}, - "d": ["foo", "bar"] - }) + c1 = Configuration({"a": {"b": 1, "c": 42}, "d": ["foo", "bar"]}) - c2 = Configuration({ - "a": {"b": 2, "e": 43}, - "d": ["fizz", "buzz"], - "f": 100 - }) + c2 = Configuration({"a": {"b": 2, "e": 43}, "d": ["fizz", "buzz"], "f": 100}) c = c1.combine(c2) assert c.get("a") == {"b": 2, "c": 42, "e": 43} @@ -33,29 +23,27 @@ def test_combine(): def test_combine_fail(): - c1 = Configuration({ - "a": "val1", - "b": "b" - }) + c1 = Configuration({"a": "val1", "b": "b"}) - c2 = Configuration({ - "a": "val2", - "c": "c" - }) + c2 = Configuration({"a": "val2", "c": "c"}) with pytest.raises(TypeError): c1.combine(c2) def test_resolve_suites(): - c = Configuration({"suites": { - "dacapo2006": { - "type": "DaCapo", - "release": "2006", - "path": "/usr/share/benchmarks/dacapo/dacapo-2006-10-MR2.jar", - "timing_iteration": 3 + c = Configuration( + { + "suites": { + "dacapo2006": { + "type": "DaCapo", + "release": "2006", + "path": "/usr/share/benchmarks/dacapo/dacapo-2006-10-MR2.jar", + "timing_iteration": 3, + } + } } - }}) + ) c.resolve_class() dacapo2006 = c.get("suites")["dacapo2006"] assert dacapo2006.release == "2006" @@ -63,13 +51,13 @@ def test_resolve_suites(): def test_resolve_modifiers(): - c = Configuration({"modifiers": { - "ss": { - "type": "EnvVar", - "var": "MMTK_PLAN", - "val": "SemiSpace" + c = Configuration( + { + "modifiers": { + "ss": {"type": "EnvVar", "var": "MMTK_PLAN", "val": "SemiSpace"} + } } - }}) + ) c.resolve_class() ss = c.get("modifiers")["ss"] assert ss.var == "MMTK_PLAN" @@ -77,15 +65,18 @@ def test_resolve_modifiers(): def test_resolve_jvms(): - c = Configuration({"runtimes": { - "adoptopenjdk-8": { - "type": "OpenJDK", - "release": "8", - "home": "/usr/lib/jvm/adoptopenjdk-8-hotspot-amd64" + c = Configuration( + { + "runtimes": { + "adoptopenjdk-8": { + "type": "OpenJDK", + "release": "8", + "home": "/usr/lib/jvm/adoptopenjdk-8-hotspot-amd64", + } + } } - }}) + ) c.resolve_class() jdk8 = c.get("runtimes")["adoptopenjdk-8"] - assert str( - jdk8.executable) == "/usr/lib/jvm/adoptopenjdk-8-hotspot-amd64/bin/java" + assert str(jdk8.executable) == "/usr/lib/jvm/adoptopenjdk-8-hotspot-amd64/bin/java" assert jdk8.release == 8 diff --git a/tests/test_fillin.py b/tests/test_fillin.py index 357ac2f..4f7d894 100644 --- a/tests/test_fillin.py +++ b/tests/test_fillin.py @@ -8,12 +8,7 @@ def test_fillin(): results = [] fillin(lambda base, ns: results.append(list(ns)), 4) - assert results == [ - [0, 8, 16], - [4, 12], - [2, 6, 10, 14], - [1, 3, 5, 7, 9, 11, 13, 15] - ] + assert results == [[0, 8, 16], [4, 12], [2, 6, 10, 14], [1, 3, 5, 7, 9, 11, 13, 15]] def test_fillin_start(): diff --git a/tests/test_log_preprocessor.py b/tests/test_log_preprocessor.py index 2d60553..daed3f9 100644 --- a/tests/test_log_preprocessor.py +++ b/tests/test_log_preprocessor.py @@ -1,37 +1,45 @@ -from running.command.log_preprocessor import filter_stats, reduce_stats, sum_work_perf_event, calc_work_ipc, ratio_work_perf_event +from running.command.log_preprocessor import ( + filter_stats, + reduce_stats, + sum_work_perf_event, + calc_work_ipc, + ratio_work_perf_event, +) def test_filter(): - assert filter_stats(lambda x: "foo" in x)({ - "foo1": 1, - "2foo": 2, - "bar3": 3, - "4bar": 4 - }) == {"foo1": 1, "2foo": 2} + assert filter_stats(lambda x: "foo" in x)( + {"foo1": 1, "2foo": 2, "bar3": 3, "4bar": 4} + ) == {"foo1": 1, "2foo": 2} def test_reduce(): - assert reduce_stats(r"\d*foo\d*", "foo_sum", lambda x, y: x + y)({ - "foo1": 1, - "2foo": 2, - "bar3": 3, - "4bar": 4 - })["foo_sum"] == 3 + assert ( + reduce_stats(r"\d*foo\d*", "foo_sum", lambda x, y: x + y)( + {"foo1": 1, "2foo": 2, "bar3": 3, "4bar": 4} + )["foo_sum"] + == 3 + ) def test_sum_perf(): - assert sum_work_perf_event("PERF_COUNT_HW_CPU_CYCLES")({ - "work.ScanUniverseRoots.PERF_COUNT_HW_CPU_CYCLES.total": 1, - "work.PrepareCollector.PERF_COUNT_HW_CPU_CYCLES.total": 2, - "work.ScanUniverseRoots.PERF_COUNT_HW_CACHE_L1D:MISS.total": 3 - })["work.PERF_COUNT_HW_CPU_CYCLES.total"] == 3 + assert ( + sum_work_perf_event("PERF_COUNT_HW_CPU_CYCLES")( + { + "work.ScanUniverseRoots.PERF_COUNT_HW_CPU_CYCLES.total": 1, + "work.PrepareCollector.PERF_COUNT_HW_CPU_CYCLES.total": 2, + "work.ScanUniverseRoots.PERF_COUNT_HW_CACHE_L1D:MISS.total": 3, + } + )["work.PERF_COUNT_HW_CPU_CYCLES.total"] + == 3 + ) def test_ratio_perf(): stats = { "work.ScanUniverseRoots.PERF_COUNT_HW_CPU_CYCLES.total": 2, "work.PrepareCollector.PERF_COUNT_HW_CPU_CYCLES.total": 3, - "work.foo.PERF_COUNT_HW_CPU_CYCLES.total": 5 + "work.foo.PERF_COUNT_HW_CPU_CYCLES.total": 5, } stats = sum_work_perf_event("PERF_COUNT_HW_CPU_CYCLES")(stats) ratios = ratio_work_perf_event("PERF_COUNT_HW_CPU_CYCLES")(stats) @@ -41,12 +49,14 @@ def test_ratio_perf(): def test_ipc(): - ipcs = calc_work_ipc({ - "work.foo.PERF_COUNT_HW_CPU_CYCLES.total": 20, - "work.foo.PERF_COUNT_HW_INSTRUCTIONS.total": 4, - "work.bar.PERF_COUNT_HW_CPU_CYCLES.total": 10, - "work.bar.PERF_COUNT_HW_INSTRUCTIONS.total": 5, - }) + ipcs = calc_work_ipc( + { + "work.foo.PERF_COUNT_HW_CPU_CYCLES.total": 20, + "work.foo.PERF_COUNT_HW_INSTRUCTIONS.total": 4, + "work.bar.PERF_COUNT_HW_CPU_CYCLES.total": 10, + "work.bar.PERF_COUNT_HW_INSTRUCTIONS.total": 5, + } + ) print(ipcs) assert ipcs["work.foo.INSTRUCTIONS_PER_CYCLE.ratio"] == 0.2 assert ipcs["work.bar.INSTRUCTIONS_PER_CYCLE.ratio"] == 0.5 diff --git a/tests/test_modifier.py b/tests/test_modifier.py index 447855e..18df727 100644 --- a/tests/test_modifier.py +++ b/tests/test_modifier.py @@ -4,48 +4,45 @@ def test_jvm_arg(): - j = JVMArg(name="j", val="-Xms100M -D\"foo bar\"") + j = JVMArg(name="j", val='-Xms100M -D"foo bar"') assert j.val == ["-Xms100M", "-Dfoo bar"] def test_jvm_classpath(): - j = JVMClasspath(name="j", val="/bin /foo \"/Users/John Citizen/\"") + j = JVMClasspath(name="j", val='/bin /foo "/Users/John Citizen/"') assert j.val == ["/bin", "/foo", "/Users/John Citizen/"] jb = JavaBenchmark( - jvm_args=[], program_args=[], cp=["fizzbuzz"], - suite_name="dacapo", name="fop" + jvm_args=[], program_args=[], cp=["fizzbuzz"], suite_name="dacapo", name="fop" ) jb = jb.attach_modifiers([j]) assert jb.cp == ["fizzbuzz", "/bin", "/foo", "/Users/John Citizen/"] def test_jvm_classpath_append(): - j = JVMClasspathAppend(name="j", val="/bin /foo \"/Users/John Citizen/\"") + j = JVMClasspathAppend(name="j", val='/bin /foo "/Users/John Citizen/"') assert j.val == ["/bin", "/foo", "/Users/John Citizen/"] jb = JavaBenchmark( - jvm_args=[], program_args=[], cp=["fizzbuzz"], - suite_name="dacapo", name="fop" + jvm_args=[], program_args=[], cp=["fizzbuzz"], suite_name="dacapo", name="fop" ) jb = jb.attach_modifiers([j]) assert jb.cp == ["fizzbuzz", "/bin", "/foo", "/Users/John Citizen/"] def test_jvm_classpath_prepend(): - j = JVMClasspathPrepend(name="j", val="/bin /foo \"/Users/John Citizen/\"") + j = JVMClasspathPrepend(name="j", val='/bin /foo "/Users/John Citizen/"') assert j.val == ["/bin", "/foo", "/Users/John Citizen/"] jb = JavaBenchmark( - jvm_args=[], program_args=[], cp=["fizzbuzz"], - suite_name="dacapo", name="fop" + jvm_args=[], program_args=[], cp=["fizzbuzz"], suite_name="dacapo", name="fop" ) jb = jb.attach_modifiers([j]) assert jb.cp == ["/bin", "/foo", "/Users/John Citizen/", "fizzbuzz"] def test_program_arg(): - p = ProgramArg(name="p", val="/bin /foo \"/Users/John Citizen/\"") + p = ProgramArg(name="p", val='/bin /foo "/Users/John Citizen/"') assert p.val == ["/bin", "/foo", "/Users/John Citizen/"] @@ -57,68 +54,49 @@ def test_expand_value_opts(): def test_modifier_set(): - c = Configuration({ - "modifiers": { - "a": { - "type": "JVMArg", - "val": "-XX:GC={0}" - }, - "b": { - "type": "EnvVar", - "var": "FOO", - "val": "BAR" - }, - "c": { - "type": "EnvVar", - "var": "FIZZ", - "val": "BUZZ" - }, - "set": { - "type": "ModifierSet", - "val": "a-{0}|b" - }, - "set_nested": { - "type": "ModifierSet", - "val": "set-{0}|c" + c = Configuration( + { + "modifiers": { + "a": {"type": "JVMArg", "val": "-XX:GC={0}"}, + "b": {"type": "EnvVar", "var": "FOO", "val": "BAR"}, + "c": {"type": "EnvVar", "var": "FIZZ", "val": "BUZZ"}, + "set": {"type": "ModifierSet", "val": "a-{0}|b"}, + "set_nested": {"type": "ModifierSet", "val": "set-{0}|c"}, } } - }) + ) c.resolve_class() - mods = c.get("modifiers")["set"].apply_value_opts( - value_opts=["NoGC"]).flatten(c) - mods = c.get("modifiers")["set_nested"].apply_value_opts( - value_opts=["NoGC"]).flatten(c) + mods = c.get("modifiers")["set"].apply_value_opts(value_opts=["NoGC"]).flatten(c) + mods = ( + c.get("modifiers")["set_nested"] + .apply_value_opts(value_opts=["NoGC"]) + .flatten(c) + ) assert len(mods) == 3 def test_no_includes_no_excludes(): - c = Configuration({ - "modifiers": { - "a": { - "type": "EnvVar", - "var": "FOO", - "val": "BAR" - } - } - }) + c = Configuration( + {"modifiers": {"a": {"type": "EnvVar", "var": "FOO", "val": "BAR"}}} + ) c.resolve_class() m = c.get("modifiers")["a"] assert m.should_attach("dacapo2006", "fop") def test_excludes(): - c = Configuration({ - "modifiers": { - "a": { - "type": "EnvVar", - "var": "FOO", - "val": "BAR", - "excludes": { - "dacapo2006": ["fop"] + c = Configuration( + { + "modifiers": { + "a": { + "type": "EnvVar", + "var": "FOO", + "val": "BAR", + "excludes": {"dacapo2006": ["fop"]}, } } } - }) + ) c.resolve_class() m = c.get("modifiers")["a"] assert not m.should_attach("dacapo2006", "fop") @@ -126,18 +104,18 @@ def test_excludes(): def test_includes(): - c = Configuration({ - "modifiers": { - "a": { - "type": "EnvVar", - "var": "FOO", - "val": "BAR", - "includes": { - "dacapo2006": ["fop"] + c = Configuration( + { + "modifiers": { + "a": { + "type": "EnvVar", + "var": "FOO", + "val": "BAR", + "includes": {"dacapo2006": ["fop"]}, } } } - }) + ) c.resolve_class() m = c.get("modifiers")["a"] assert m.should_attach("dacapo2006", "fop") @@ -145,21 +123,19 @@ def test_includes(): def test_includes_excludes(): - c = Configuration({ - "modifiers": { - "a": { - "type": "EnvVar", - "var": "FOO", - "val": "BAR", - "includes": { - "dacapo2006": ["fop", "hsqldb"] - }, - "excludes": { - "dacapo2006": ["fop"] + c = Configuration( + { + "modifiers": { + "a": { + "type": "EnvVar", + "var": "FOO", + "val": "BAR", + "includes": {"dacapo2006": ["fop", "hsqldb"]}, + "excludes": {"dacapo2006": ["fop"]}, } } } - }) + ) c.resolve_class() m = c.get("modifiers")["a"] assert not m.should_attach("dacapo2006", "fop") diff --git a/tests/test_runbms.py b/tests/test_runbms.py index 45e0626..25057cb 100644 --- a/tests/test_runbms.py +++ b/tests/test_runbms.py @@ -13,7 +13,8 @@ def test_spread_1(): spread_factor = 1 N = 8 for i in range(1, N + 1): - left = pytest.approx(spread(spread_factor, N, i) - - spread(spread_factor, N, i - 1)) - right = pytest.approx(1 + (i-1) / 7) + left = pytest.approx( + spread(spread_factor, N, i) - spread(spread_factor, N, i - 1) + ) + right = pytest.approx(1 + (i - 1) / 7) assert left == right diff --git a/tests/test_suite.py b/tests/test_suite.py index b312d8f..f5c2215 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -5,44 +5,38 @@ def test_binary_benchmark_suite_quoted(): b = BinaryBenchmarkSuite( - name="foobar", - programs={ - "ls": { - "path": "/bin/ls", - "args": "foo \"bar buzz\"" - } - } + name="foobar", programs={"ls": {"path": "/bin/ls", "args": 'foo "bar buzz"'}} ) assert b.programs["ls"]["args"] == ["foo", "bar buzz"] def test_dacapo_size(): - c = Configuration({ - "suites": { - "dacapo2006": { - "type": "DaCapo", - "release": "2006", - "path": "/usr/share/benchmarks/dacapo/dacapo-2006-10-MR2.jar", - "timing_iteration": 3 + c = Configuration( + { + "suites": { + "dacapo2006": { + "type": "DaCapo", + "release": "2006", + "path": "/usr/share/benchmarks/dacapo/dacapo-2006-10-MR2.jar", + "timing_iteration": 3, + }, + "dacapo2006_default": { + "type": "DaCapo", + "release": "2006", + "path": "/usr/share/benchmarks/dacapo/dacapo-2006-10-MR2.jar", + "timing_iteration": 3, + "size": "default", + }, + }, + "benchmarks": { + "dacapo2006": [ + "fop", + dict(name="fop_small", bm_name="fop", size="small"), + ], + "dacapo2006_default": ["fop"], }, - "dacapo2006_default": { - "type": "DaCapo", - "release": "2006", - "path": "/usr/share/benchmarks/dacapo/dacapo-2006-10-MR2.jar", - "timing_iteration": 3, - "size": "default" - } - }, - "benchmarks": { - "dacapo2006": [ - "fop", - dict(name="fop_small", bm_name="fop", size="small") - ], - "dacapo2006_default": [ - "fop" - ] } - }) + ) c.resolve_class() fop = c.get("benchmarks")["dacapo2006"][0] @@ -54,23 +48,26 @@ def test_dacapo_size(): def test_dacapo_timing_iteration(): - c = Configuration({ - "suites": { - "dacapo2006": { - "type": "DaCapo", - "release": "2006", - "path": "/usr/share/benchmarks/dacapo/dacapo-2006-10-MR2.jar", - "timing_iteration": 3 - } - }, - "benchmarks": { - "dacapo2006": [ - "fop", - dict(name="fop_converge", bm_name="fop", - timing_iteration="converge") - ] + c = Configuration( + { + "suites": { + "dacapo2006": { + "type": "DaCapo", + "release": "2006", + "path": "/usr/share/benchmarks/dacapo/dacapo-2006-10-MR2.jar", + "timing_iteration": 3, + } + }, + "benchmarks": { + "dacapo2006": [ + "fop", + dict( + name="fop_converge", bm_name="fop", timing_iteration="converge" + ), + ] + }, } - }) + ) c.resolve_class() fop = c.get("benchmarks")["dacapo2006"][0] @@ -80,39 +77,35 @@ def test_dacapo_timing_iteration(): def test_dacapo_openjdk_9_workaround(): - c = Configuration({ - "suites": { - "dacapo2006": { - "type": "DaCapo", - "release": "2006", - "path": "/usr/share/benchmarks/dacapo/dacapo-2006-10-MR2.jar", - "timing_iteration": 3 - } - }, - "benchmarks": { - "dacapo2006": [ - "fop" - ] - }, - "runtimes": { - "jdk8": { - "type": "OpenJDK", - "release": 8, - "home": "/usr/lib/jvm/temurin-8-jdk-amd64" + c = Configuration( + { + "suites": { + "dacapo2006": { + "type": "DaCapo", + "release": "2006", + "path": "/usr/share/benchmarks/dacapo/dacapo-2006-10-MR2.jar", + "timing_iteration": 3, + } }, - "jdk11": { - "type": "OpenJDK", - "release": 11, - "home": "/usr/lib/jvm/temurin-11-jdk-amd64" + "benchmarks": {"dacapo2006": ["fop"]}, + "runtimes": { + "jdk8": { + "type": "OpenJDK", + "release": 8, + "home": "/usr/lib/jvm/temurin-8-jdk-amd64", + }, + "jdk11": { + "type": "OpenJDK", + "release": 11, + "home": "/usr/lib/jvm/temurin-11-jdk-amd64", + }, }, - }, - "configs": [ - "jdk8", - "jdk11" - ] - }) + "configs": ["jdk8", "jdk11"], + } + ) c.resolve_class() from running.benchmark import JavaBenchmark + fop: JavaBenchmark fop = c.get("benchmarks")["dacapo2006"][0] jdk8 = c.get("runtimes")["jdk8"] diff --git a/tests/test_util.py b/tests/test_util.py index 528566d..f83d4bc 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -4,37 +4,35 @@ def test_split_quoted(): - assert split_quoted("123 \"foo bar\"") == ["123", "foo bar"] + assert split_quoted('123 "foo bar"') == ["123", "foo bar"] def test_smart_quote(): - assert smart_quote(Path("/bin")/"123 456") == "\"/bin/123 456\"" + assert smart_quote(Path("/bin") / "123 456") == '"/bin/123 456"' def test_issue104(): - c = Configuration({ - "suites": { - "dacapo2006": { - "type": "DaCapo", - "release": "2006", - "path": "/usr/share/benchmarks/dacapo/dacapo-2006-10-MR2.jar", - "timing_iteration": 3 - } - }, - "benchmarks": { - "dacapo2006": [ - "fop" - ] - }, - "runtimes": { - "jdk8": { - "type": "OpenJDK", - "release": 8, - "home": "/usr/lib/jvm/temurin-8-jdk-amd64" - } - }, - "modifiers": {} - }) + c = Configuration( + { + "suites": { + "dacapo2006": { + "type": "DaCapo", + "release": "2006", + "path": "/usr/share/benchmarks/dacapo/dacapo-2006-10-MR2.jar", + "timing_iteration": 3, + } + }, + "benchmarks": {"dacapo2006": ["fop"]}, + "runtimes": { + "jdk8": { + "type": "OpenJDK", + "release": 8, + "home": "/usr/lib/jvm/temurin-8-jdk-amd64", + } + }, + "modifiers": {}, + } + ) c.resolve_class() _, modifiers = parse_config_str(c, "jdk8|") assert len(modifiers) == 0