From 434e9cb9a65e6df5d12ece6ee7cdacecc60e122e Mon Sep 17 00:00:00 2001 From: agrecu Date: Sat, 17 Feb 2024 04:02:25 +0200 Subject: [PATCH 1/7] Implement autoconfig generation and improve platform name compression --- run.py | 60 ++++++++++++++++++++++++++++++++++++-- status_checker.py | 73 +++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 121 insertions(+), 12 deletions(-) diff --git a/run.py b/run.py index bc55b68..0867a65 100644 --- a/run.py +++ b/run.py @@ -4,6 +4,14 @@ from datetime import date as dt from datetime import datetime +try: + import config as cfg +except ImportError as e: + class cfg(object): + slots_to_check = StatusChecker.slots_to_check + platforms_to_check = StatusChecker.platforms_to_check + projects_to_check = StatusChecker.projects_to_check + @click.group() @click.option("--verbosity", default="INFO", help="verbosity of the logger") @@ -20,19 +28,19 @@ def common(func): )(func) func = click.option( "--slots", - default=StatusChecker.slots_to_check, + default=cfg.slots_to_check, help="list of nightly slot names to check", multiple=True, )(func) func = click.option( "--platforms", - default=StatusChecker.platforms_to_check, + default=cfg.platforms_to_check, help="list of platform names to check", multiple=True, )(func) func = click.option( "--projects", - default=StatusChecker.projects_to_check, + default=cfg.projects_to_check, help="list of project names to check", multiple=True, )(func) @@ -99,8 +107,54 @@ def dqcs_report( ) +@click.command() +@common +def mkconfig(date, slots, platforms, projects): + """Generate config.py to customize selection of slots, platforms and projects.""" + cfg_code = """# Module to customize default configuration for nightly-status-checker + +slots_to_check = [ + {slots_list} +] + +projects_to_check = [ + {project_list} +] + +platforms_to_check = [ + {platform_list} +] +""" + pretty_sep = ",\n " + slots_list = [] + project_list = [] + platform_list = [] + checker = StatusChecker( + slot_names=slots, + platform_names=platforms, + project_names=projects, + ) + slots_list = [sn for sn in checker._slots.keys()] + miss_slots = [sn for sn in StatusChecker.slots_to_check if sn not in slots_list] + if len(miss_slots) > 0: + logging.warning("Hardcoded default slots {} not found in Nightly page. Maybe update package!".format( + ', '.join(miss_slots))) + for slot, build in checker._slots.items(): + r = checker._get_Platforms_Projects_for_slot(slot, build) + project_list += [pn for pn in r[1] if pn not in project_list] + platform_list += [pn for pn in r[0] if pn not in platform_list] + slots_str = pretty_sep.join(['"{}"'.format(ss) for ss in slots_list]) + projects_str = pretty_sep.join(['"{}"'.format(ss) for ss in project_list]) + platforms_str = pretty_sep.join(['"{}"'.format(ss) for ss in platform_list]) + with open("config.py", 'w', encoding='utf-8') as fp: + fp.write(cfg_code.format(slots_list=slots_str, project_list=projects_str, platform_list=platforms_str)) + fp.flush() + logging.info("'config.py' file written to disk. Edit accordingly and run script again for desired function.") + + cli.add_command(current_status) cli.add_command(dqcs_report) +cli.add_command(mkconfig) if __name__ == "__main__": cli() diff --git a/status_checker.py b/status_checker.py index 96bd568..58e7460 100644 --- a/status_checker.py +++ b/status_checker.py @@ -15,6 +15,19 @@ ) +def tokenizePlatforms(plist): + tree = {} + for pp in set(plist): + toks = pp.split('-', 3) + for i in range(0, 4): + tk = '-'.join(toks[0:i]) + if tk in tree: + tree[tk].append(pp) + else: + tree[tk] = [pp, ] + return tree + + class StatusChecker: slots_to_check = [ "lhcb-sim10-dev", @@ -79,6 +92,7 @@ def __init__( slot_names: list = [], platform_names: list = [], project_names: list = [], + cmd_mk_config = False, ): if slot_names: self.slots_to_check = slot_names @@ -87,6 +101,8 @@ def __init__( if project_names: self.projects_to_check = project_names self.get_current_builds() + self._tkPlatforms = tokenizePlatforms(self.platforms_to_check) + logging.debug("Tokens " + str(self._tkPlatforms)) @request def get_current_builds(self): @@ -114,6 +130,40 @@ def get_current_builds(self): self._slots[slot] = build_id logging.debug(f"Found build ids: {dict(self._slots)}.") + def _get_short_platforms_for_results(self, plist): + """Return a list of short platform names for platforms to check in results. + Replace common prefixes by * referring to previous platform considered.""" + ret = [] + for pc in plist: + if len(ret) == 0: + ret.append(pc) + pp = pc + continue + pk = "" + for kk, lst in self._tkPlatforms.items(): + if pp in lst and pc in lst and len(kk) > len(pk): + pk = kk + if len(pk) == 0: + ret.append(pc) + else: + ret.append('*' + pc[len(pk):]) + pp = pc + return ret + + def _get_Platforms_Projects_for_slot(self, slot: str, build_id: int) -> ([], []): + response = requests.get(f"{self.api_page}/{slot}/{build_id}/summary") + response.raise_for_status() + parsed = response.json() + platforms = [] + projects = [] + if parsed["aborted"]: + return platforms, projects + if 'platforms' in parsed: + platforms = parsed['platforms'] + if 'projects' in parsed: + projects = [pdic['name'] for pdic in parsed['projects'] if pdic['enabled']] + return platforms, projects + def _fetch_build_info( self, slot: str, @@ -128,18 +178,22 @@ def _fetch_build_info( return df, parsed_date errors_summary = defaultdict(lambda: 0) failed_summary = defaultdict(lambda: 0) + long_platforms = [] for project in parsed["projects"]: if ( project["name"] in self.projects_to_check and project["enabled"] ): if df.empty: - short_platforms = [ - # platform.replace(self.hidden_platform_prefix, "*") - re.sub(self.hidden_platform_prefix_re, "*", platform) - for platform in self.platforms_to_check - if platform in project["results"] - ] + long_platforms = [pn for pn in self.platforms_to_check if pn in project['results']] + long_platforms.sort(reverse=True) + short_platforms = self._get_short_platforms_for_results(long_platforms) + # short_platforms = [ + # # platform.replace(self.hidden_platform_prefix, "*") + # re.sub(self.hidden_platform_prefix_re, "*", platform) + # for platform in self.platforms_to_check + # if platform in project["results"] + # ] nested_results_cols = [("Project", ""), ("Failed MRs", "")] nested_results_cols += [ (platform, "BUILD / TEST") @@ -149,9 +203,10 @@ def _fetch_build_info( columns=pd.MultiIndex.from_tuples(nested_results_cols) ) ptf_res = [] - for platform in self.platforms_to_check: - if platform not in project["results"]: - continue + # for platform in self.platforms_to_check: + for platform in long_platforms: + # if platform not in project["results"]: + # continue results = project["results"][platform] tmp_res = [] for check_type, check_values in self.result_types.items(): From 0f4b937d6ec034581a9844dae00ec2e5d8c0dce5 Mon Sep 17 00:00:00 2001 From: agrecu Date: Sat, 17 Feb 2024 10:01:24 +0200 Subject: [PATCH 2/7] Implement autoconfig generation and improve platform name compression --- run.py | 29 ++++++++++++++++++----------- status_checker.py | 16 +++++++++------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/run.py b/run.py index 0867a65..e951fcb 100644 --- a/run.py +++ b/run.py @@ -6,7 +6,7 @@ try: import config as cfg -except ImportError as e: +except ImportError: class cfg(object): slots_to_check = StatusChecker.slots_to_check platforms_to_check = StatusChecker.platforms_to_check @@ -111,8 +111,8 @@ def dqcs_report( @common def mkconfig(date, slots, platforms, projects): """Generate config.py to customize selection of slots, platforms and projects.""" - cfg_code = """# Module to customize default configuration for nightly-status-checker - + cfg_code = """# Module to customize default config of nightly-status-checker + slots_to_check = [ {slots_list} ] @@ -135,21 +135,28 @@ def mkconfig(date, slots, platforms, projects): project_names=projects, ) slots_list = [sn for sn in checker._slots.keys()] - miss_slots = [sn for sn in StatusChecker.slots_to_check if sn not in slots_list] + miss_slots = [sn for sn in StatusChecker.slots_to_check + if sn not in slots_list] if len(miss_slots) > 0: - logging.warning("Hardcoded default slots {} not found in Nightly page. Maybe update package!".format( - ', '.join(miss_slots))) + logging.warning("Hardcoded default slots {} not found in Nightly page." + " Maybe update package!".format(', '.join(miss_slots))) for slot, build in checker._slots.items(): r = checker._get_Platforms_Projects_for_slot(slot, build) project_list += [pn for pn in r[1] if pn not in project_list] platform_list += [pn for pn in r[0] if pn not in platform_list] - slots_str = pretty_sep.join(['"{}"'.format(ss) for ss in slots_list]) - projects_str = pretty_sep.join(['"{}"'.format(ss) for ss in project_list]) - platforms_str = pretty_sep.join(['"{}"'.format(ss) for ss in platform_list]) + slots_str = pretty_sep.join(['"{}"'.format(ss) + for ss in slots_list]) + projects_str = pretty_sep.join(['"{}"'.format(ss) + for ss in project_list]) + platforms_str = pretty_sep.join(['"{}"'.format(ss) + for ss in platform_list]) with open("config.py", 'w', encoding='utf-8') as fp: - fp.write(cfg_code.format(slots_list=slots_str, project_list=projects_str, platform_list=platforms_str)) + fp.write(cfg_code.format(slots_list=slots_str, + project_list=projects_str, + platform_list=platforms_str)) fp.flush() - logging.info("'config.py' file written to disk. Edit accordingly and run script again for desired function.") + logging.info("'config.py' file written to disk. " + "Edit accordingly and run script again for desired function.") cli.add_command(current_status) diff --git a/status_checker.py b/status_checker.py index 58e7460..27ab213 100644 --- a/status_checker.py +++ b/status_checker.py @@ -130,9 +130,9 @@ def get_current_builds(self): self._slots[slot] = build_id logging.debug(f"Found build ids: {dict(self._slots)}.") - def _get_short_platforms_for_results(self, plist): - """Return a list of short platform names for platforms to check in results. - Replace common prefixes by * referring to previous platform considered.""" + def _get_short_platforms(self, plist): + """Return list of short platform names to check in results. + Replace common prefixes by * w.r.t. previous platform considered.""" ret = [] for pc in plist: if len(ret) == 0: @@ -150,7 +150,7 @@ def _get_short_platforms_for_results(self, plist): pp = pc return ret - def _get_Platforms_Projects_for_slot(self, slot: str, build_id: int) -> ([], []): + def _get_Platforms_Projects_for_slot(self, slot: str, build_id: int): response = requests.get(f"{self.api_page}/{slot}/{build_id}/summary") response.raise_for_status() parsed = response.json() @@ -161,7 +161,8 @@ def _get_Platforms_Projects_for_slot(self, slot: str, build_id: int) -> ([], []) if 'platforms' in parsed: platforms = parsed['platforms'] if 'projects' in parsed: - projects = [pdic['name'] for pdic in parsed['projects'] if pdic['enabled']] + projects = [pdic['name'] for pdic in parsed['projects'] + if pdic['enabled']] return platforms, projects def _fetch_build_info( @@ -185,9 +186,10 @@ def _fetch_build_info( and project["enabled"] ): if df.empty: - long_platforms = [pn for pn in self.platforms_to_check if pn in project['results']] + long_platforms = [pn for pn in self.platforms_to_check + if pn in project['results']] long_platforms.sort(reverse=True) - short_platforms = self._get_short_platforms_for_results(long_platforms) + short_platforms = self._get_short_platforms(long_platforms) # short_platforms = [ # # platform.replace(self.hidden_platform_prefix, "*") # re.sub(self.hidden_platform_prefix_re, "*", platform) From 25364974c046e3d50f0c638ad0772d07e7d87b58 Mon Sep 17 00:00:00 2001 From: agrecu Date: Sat, 17 Feb 2024 10:48:52 +0200 Subject: [PATCH 3/7] Implement autoconfig generation and improve platform name compression --- run.py | 6 ++++-- status_checker.py | 19 +++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/run.py b/run.py index e951fcb..5e5439d 100644 --- a/run.py +++ b/run.py @@ -110,8 +110,10 @@ def dqcs_report( @click.command() @common def mkconfig(date, slots, platforms, projects): - """Generate config.py to customize selection of slots, platforms and projects.""" - cfg_code = """# Module to customize default config of nightly-status-checker + """Generate config.py to customize + selection of slots, platforms and projects.""" + cfg_code = """ +# Module to customize default configuration of nightly-status-checker slots_to_check = [ {slots_list} diff --git a/status_checker.py b/status_checker.py index 27ab213..ea93296 100644 --- a/status_checker.py +++ b/status_checker.py @@ -89,10 +89,9 @@ class StatusChecker: def __init__( self, - slot_names: list = [], - platform_names: list = [], - project_names: list = [], - cmd_mk_config = False, + slot_names: list = (), + platform_names: list = (), + project_names: list = (), ): if slot_names: self.slots_to_check = slot_names @@ -118,7 +117,7 @@ def get_current_builds(self): msg = ( f"No slots from the list '{self.slots_to_check}' " f"were found in the content of '{self.main_page}'. " - "Please, make sure you provided correct slot names." + f"Please, make sure you provided correct slot names." ) logging.error(msg) raise ValueError(msg) @@ -134,6 +133,7 @@ def _get_short_platforms(self, plist): """Return list of short platform names to check in results. Replace common prefixes by * w.r.t. previous platform considered.""" ret = [] + pp = None for pc in plist: if len(ret) == 0: ret.append(pc) @@ -150,7 +150,10 @@ def _get_short_platforms(self, plist): pp = pc return ret - def _get_Platforms_Projects_for_slot(self, slot: str, build_id: int): + def _get_Platforms_Projects_for_slot(self, + slot: str, + build_id: int, + ) -> ([], []): response = requests.get(f"{self.api_page}/{slot}/{build_id}/summary") response.raise_for_status() parsed = response.json() @@ -363,11 +366,11 @@ def check_status( logging.warning( f" Found in total {counter} ERRORs in " f"BUILDING the project '{project}'. " - "Verify this and report if this is not known." + f"Verify this and report if this is not known." ) for project, counter in failed_summary.items(): logging.warning( f" Found in total {counter} FAILED TESTs in " f"the project '{project}'. " - "Verify this and report if this is not known." + f"Verify this and report if this is not known." ) From f3da0f481ea291814cfcbf9b99b8becad71e46f0 Mon Sep 17 00:00:00 2001 From: agrecu Date: Sat, 17 Feb 2024 12:19:38 +0200 Subject: [PATCH 4/7] Implement autoconfig generation and improve platform name compression --- status_checker.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/status_checker.py b/status_checker.py index ea93296..53c54cf 100644 --- a/status_checker.py +++ b/status_checker.py @@ -150,10 +150,11 @@ def _get_short_platforms(self, plist): pp = pc return ret - def _get_Platforms_Projects_for_slot(self, - slot: str, - build_id: int, - ) -> ([], []): + def _get_Platforms_Projects_for_slot( + self, + slot: str, + build_id: int, + ) -> ([], []): response = requests.get(f"{self.api_page}/{slot}/{build_id}/summary") response.raise_for_status() parsed = response.json() From 515bffe260696d8d2bfaa386f8bc2bc9637d4c91 Mon Sep 17 00:00:00 2001 From: agrecu Date: Sat, 17 Feb 2024 13:10:13 +0200 Subject: [PATCH 5/7] Implement autoconfig generation and improve platform name compression --- status_checker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/status_checker.py b/status_checker.py index 53c54cf..c3a94a2 100644 --- a/status_checker.py +++ b/status_checker.py @@ -129,7 +129,10 @@ def get_current_builds(self): self._slots[slot] = build_id logging.debug(f"Found build ids: {dict(self._slots)}.") - def _get_short_platforms(self, plist): + def _get_short_platforms( + self, + plist: [], + ) -> []: """Return list of short platform names to check in results. Replace common prefixes by * w.r.t. previous platform considered.""" ret = [] From 2f008f16c4dd759a50a6a81c14820929a3ebb0e4 Mon Sep 17 00:00:00 2001 From: agrecu Date: Sun, 18 Feb 2024 03:39:32 +0200 Subject: [PATCH 6/7] Implement autoconfig generation and improve platform name compression --- status_checker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/status_checker.py b/status_checker.py index c3a94a2..683935e 100644 --- a/status_checker.py +++ b/status_checker.py @@ -149,7 +149,8 @@ def _get_short_platforms( if len(pk) == 0: ret.append(pc) else: - ret.append('*' + pc[len(pk):]) + ss = slice(len(pk), len(pc)) + ret.append('*' + pc[ss]) pp = pc return ret From 2c9907e2285159cb964d6dd3510aaaa8df2528b2 Mon Sep 17 00:00:00 2001 From: agrecu Date: Sun, 18 Feb 2024 13:35:54 +0200 Subject: [PATCH 7/7] Make short tags name unique for PANDAs --- run.py | 1 - status_checker.py | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/run.py b/run.py index 5e5439d..8636c1c 100644 --- a/run.py +++ b/run.py @@ -128,7 +128,6 @@ def mkconfig(date, slots, platforms, projects): ] """ pretty_sep = ",\n " - slots_list = [] project_list = [] platform_list = [] checker = StatusChecker( diff --git a/status_checker.py b/status_checker.py index 683935e..cbd1336 100644 --- a/status_checker.py +++ b/status_checker.py @@ -204,6 +204,20 @@ def _fetch_build_info( # for platform in self.platforms_to_check # if platform in project["results"] # ] + # make short platform names unique for PANDA (add +1,+2, ... to same names in list) + ssplatforms = set(short_platforms) + if len(ssplatforms) < len(short_platforms): + for pn in ssplatforms: + if not pn.startswith('*') or short_platforms.count(pn) == 1: + continue + pc = 1 + while True: + try: + ip = short_platforms.index(pn) + short_platforms[ip] += '!{}'.format(pc) + pc += 1 + except ValueError: + break nested_results_cols = [("Project", ""), ("Failed MRs", "")] nested_results_cols += [ (platform, "BUILD / TEST")