From bbc383d79ab7e0b7a38f310458b9c6b886ebfaaf Mon Sep 17 00:00:00 2001 From: Andrija Kolic Date: Thu, 21 Mar 2024 13:27:53 +0100 Subject: [PATCH] Change default tracker to solely 'rsspercentiles'. Add rule that copies the 99th rss percentile into max-rss for a seamless transition. Cherry-pick of previously reverted commit 374d53703cab748a152364d3fdf6a08b4f988f7d. --- src/mx/_impl/mx.py | 2 +- src/mx/_impl/mx_benchmark.py | 57 +++++++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/src/mx/_impl/mx.py b/src/mx/_impl/mx.py index e97692e5..5c24f689 100755 --- a/src/mx/_impl/mx.py +++ b/src/mx/_impl/mx.py @@ -18147,7 +18147,7 @@ def alarm_handler(signum, frame): _CACHE_DIR = get_env('MX_CACHE_DIR', join(dot_mx_dir(), 'cache')) # The version must be updated for every PR (checked in CI) and the comment should reflect the PR's issue -version = VersionSpec("7.22.1") # [GR-53101] Remove possibility of having a list of trackers +version = VersionSpec("7.22.2") # [GR-51382] Replace maxrss in favor of rsspercentiles as the default tracker _mx_start_datetime = datetime.utcnow() diff --git a/src/mx/_impl/mx_benchmark.py b/src/mx/_impl/mx_benchmark.py index 56f49402..a98a885e 100644 --- a/src/mx/_impl/mx_benchmark.py +++ b/src/mx/_impl/mx_benchmark.py @@ -121,6 +121,7 @@ from argparse import SUPPRESS from collections import OrderedDict from typing import Callable, Sequence, Iterable, NoReturn, Optional, Dict, Any, List, Collection +from .support.logging import log_deprecation from . import mx @@ -2726,6 +2727,10 @@ def get_rules(self, bmSuiteArgs): raise NotImplementedError() class RssTracker(Tracker): + def __init__(self, bmSuite): + super().__init__(bmSuite) + log_deprecation("The 'rss' tracker is deprecated, use 'rsspercentiles' instead!") + def map_command(self, cmd): """ Tracks the max resident memory size used by the process using the 'time' command. @@ -2891,10 +2896,12 @@ class RssPercentilesTracker(Tracker): # the time period between two polls, in seconds poll_interval = 0.1 - def __init__(self, bmSuite, skip=0): + def __init__(self, bmSuite, skip=0, copy_into_max_rss=True): super().__init__(bmSuite) self.most_recent_text_output = None self.skip = skip # the number of RSS entries to skip from each poll (used to skip entries of other trackers) + self.copy_into_max_rss = copy_into_max_rss + self.percentile_data_points = [] def map_command(self, cmd): if not _use_tracker: @@ -2908,6 +2915,7 @@ def map_command(self, cmd): bench_name = self.bmSuite.currently_running_benchmark() if self.bmSuite else "benchmark" if self.bmSuite: bench_name = f"{self.bmSuite.name()}-{bench_name}" + bench_name = bench_name.replace("/", "-") ts = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") text_output = os.path.join(os.getcwd(), f"ps_{bench_name}_{ts}.txt") @@ -2916,7 +2924,10 @@ def map_command(self, cmd): return ["python3", ps_poller_script_path, "-f", text_output, "-i", str(RssPercentilesTracker.poll_interval)] + cmd def get_rules(self, bmSuiteArgs): - return [RssPercentilesTracker.RssPercentilesRule(self, bmSuiteArgs)] + if self.copy_into_max_rss: + return [RssPercentilesTracker.RssPercentilesRule(self, bmSuiteArgs), RssPercentilesTracker.MaxRssCopyRule(self, bmSuiteArgs)] + else: + return [RssPercentilesTracker.RssPercentilesRule(self, bmSuiteArgs)] class RssPercentilesRule(CSVBaseRule): def __init__(self, tracker, bmSuiteArgs, **kwargs): @@ -2986,17 +2997,49 @@ def pc(k): # k-percentile with linear interpolation between closest ranks v = v / 1024 # convert to MB return {"metric_percentile": str(k), "metric_value": str(int(v))} - percentiles = [pc(perc) for perc in RssPercentilesTracker.interesting_percentiles] - for rss_percentile in percentiles: + self.tracker.percentile_data_points = [pc(perc) for perc in RssPercentilesTracker.interesting_percentiles] + for rss_percentile in self.tracker.percentile_data_points: mx.log(f"\t{rss_percentile['metric_percentile']}th RSS percentile (MB): {rss_percentile['metric_value']}") - return percentiles + return self.tracker.percentile_data_points + + class MaxRssCopyRule(BaseRule): + percentile_to_copy_into_max_rss = 99 + + def __init__(self, tracker, bmSuiteArgs): + replacement = { + "benchmark": tracker.bmSuite.currently_running_benchmark(), + "bench-suite": tracker.bmSuite.benchSuiteName(bmSuiteArgs) if mx_benchmark_compatibility().bench_suite_needs_suite_args() else tracker.bmSuite.benchSuiteName(), + "config.vm-flags": ' '.join(tracker.bmSuite.vmArgs(bmSuiteArgs)), + "metric.name": "max-rss", + "metric.value": ("", int), + "metric.unit": "MB", + "metric.type": "numeric", + "metric.score-function": "id", + "metric.better": "lower", + "metric.iteration": 0 + } + super().__init__(replacement) + self.tracker = tracker + + def parseResults(self, text): + for rss_dp in self.tracker.percentile_data_points: + if rss_dp['metric_percentile'] == str(RssPercentilesTracker.MaxRssCopyRule.percentile_to_copy_into_max_rss): + mx.log(f"\n\tmax-rss copied from {rss_dp['metric_percentile']}th RSS percentile (MB): {rss_dp['metric_value']}") + return [{"metric_value": rss_dp['metric_value']}] + mx.warn(f"Couldn't find {RssPercentilesTracker.MaxRssCopyRule.percentile_to_copy_into_max_rss}th RSS percentile to copy into max-rss, metric will be omitted!") + return [] class RssPercentilesAndMaxTracker(Tracker): def __init__(self, bmSuite): super().__init__(bmSuite) self.rss_max_tracker = RssTracker(bmSuite) - self.rss_percentiles_tracker = RssPercentilesTracker(bmSuite, skip=1) # skip RSS of the 'time' command + self.rss_percentiles_tracker = RssPercentilesTracker( + bmSuite, + skip=1, # skip RSS of the 'time' command + copy_into_max_rss=False # don't copy a percentile into max-rss, since the rss_max_tracker will generate the metric + ) + log_deprecation("The 'rsspercentiles+maxrss' tracker is deprecated, use 'rsspercentiles' instead!") def map_command(self, cmd): return self.rss_percentiles_tracker.map_command(self.rss_max_tracker.map_command(cmd)) @@ -3288,7 +3331,7 @@ def benchmark(self, mxBenchmarkArgs, bmSuiteArgs, returnSuiteAndResults=False): parser.add_argument( "--bench-suite-version", default=None, help="Desired version of the benchmark suite to execute.") parser.add_argument( - "--tracker", default='rss', help="Enable extra trackers like 'rss' or 'psrecord'. If not set, 'rss' is used.") + "--tracker", default='rsspercentiles', help="Enable extra trackers like 'rsspercentiles', 'rss' or 'psrecord'. If not set, 'rsspercentiles' is used.") parser.add_argument( "--machine-name", default=None, help="Abstract name of the target machine.") parser.add_argument(