From f760f26a6e363aecc5b892f1fc6e44e72ebfe141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mazurek?= Date: Wed, 22 Nov 2023 15:40:18 +0100 Subject: [PATCH] Formatting and add workflows :broom: --- .github/workflows/pre-commit.yaml | 29 ++++++++++++++ .github/workflows/tests.yaml | 26 +++++++++++++ .pre-commit-config.yaml | 10 +++++ pyproject.toml | 2 + status_checker.py | 65 +++++++++++++++++++------------ test/__init__.py | 0 test/test_status_checker.py | 17 ++++++++ 7 files changed, 125 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/pre-commit.yaml create mode 100644 .github/workflows/tests.yaml create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml create mode 100644 test/__init__.py create mode 100644 test/test_status_checker.py diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 0000000..3c6f53e --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,29 @@ + +name: Pre-commit + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + pre-commit: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: 3.9.18 + + - name: Install pre-commit + run: pip install pre-commit + + - name: Run pre-commit + run: pre-commit run --all-files diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..c202c6e --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,26 @@ +name: Run pytests + +on: [push] + +jobs: + build-linux: + runs-on: ubuntu-latest + strategy: + max-parallel: 5 + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.9.18 + uses: actions/setup-python@v3 + with: + python-version: '3.9.18' + - name: Add conda to system path + run: | + # $CONDA is an environment variable pointing to the root of the miniconda directory + echo $CONDA/bin >> $GITHUB_PATH + - name: Install dependencies + run: | + conda env update --file environment.yml --name base + - name: Test with pytest + run: | + pytest test \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..51f0952 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,10 @@ +repos: +- repo: https://github.com/psf/black + rev: 23.9.1 + hooks: + - id: black + language_version: python3.11 +- repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9216134 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.black] +line-length = 79 \ No newline at end of file diff --git a/status_checker.py b/status_checker.py index 7dc7d0f..9bfab4f 100644 --- a/status_checker.py +++ b/status_checker.py @@ -16,7 +16,6 @@ class StatusChecker: - slots_to_check = [ "lhcb-sim10-dev", "lhcb-sim10", @@ -62,7 +61,7 @@ class StatusChecker: } hidden_platform_prefix = "x86_64_v2-centos7-gcc11" - hidden_platform_prefix_re = r'x86_64_v2(-centos7)?(-gcc11)?' + hidden_platform_prefix_re = r"x86_64_v2(-centos7)?(-gcc11)?" # there is no way you can get the list of build ids # from the API, so we have to use the main page... @@ -97,7 +96,9 @@ def get_current_builds(self): response.raise_for_status() slots_reg = "|".join(self.slots_to_check) slots_reg = rf"(?:{slots_reg})\/[0-9]{{1,4}}\/" - slot_candidates = re.findall(slots_reg, response.content.decode("utf-8")) + slot_candidates = re.findall( + slots_reg, response.content.decode("utf-8") + ) if not slot_candidates: msg = ( f"No slots from the list '{self.slots_to_check}' " @@ -127,17 +128,21 @@ def _fetch_build_info( if parsed["aborted"]: return df, parsed_date for project in parsed["projects"]: - if project["name"] in self.projects_to_check and project["enabled"]: + if ( + project["name"] in self.projects_to_check + and project["enabled"] + ): if df.empty: short_platforms = [ # platform.replace(self.hidden_platform_prefix, "*") - re.sub(self.hidden_platform_prefix_re, '*', platform) + re.sub(self.hidden_platform_prefix_re, "*", platform) for platform in self.platforms_to_check if platform in project["results"] ] nested_results_cols = [("Project", ""), ("Failed MRs", "")] nested_results_cols += [ - (platform, "BUILD / TEST") for platform in short_platforms + (platform, "BUILD / TEST") + for platform in short_platforms ] df = pd.DataFrame( columns=pd.MultiIndex.from_tuples(nested_results_cols) @@ -156,16 +161,15 @@ def _fetch_build_info( f"{self.parsed_result_type[result_name]}" f"{str(results[check_type][result_name])}" ) - except KeyError: - logging.debug( - f'Missing key [{check_type}][{result_name}] in {results}. Output will be incomplete') - tmp_tmp_res = ["UNKNOWN"] - break except TypeError: tmp_tmp_res = ["UNKNOWN"] break except KeyError: - logging.debug(f'Missing key [{check_type}][{result_name}] in {results}. Output will be incomplete') + logging.debug( + f"Missing key [{check_type}]" + f"[{result_name}] in {results}. " + "Output will be incomplete" + ) tmp_tmp_res = ["UNKNOWN"] break tmp_res.append(" ".join(tmp_tmp_res)) @@ -173,8 +177,13 @@ def _fetch_build_info( failed_MRs = [] if project["checkout"] and "warnings" in project["checkout"]: for warn in project["checkout"]["warnings"]: - tmp_res = re.findall(rf"{project['name']}\![0-9]{{1,5}}", warn) - failed_MRs += [r.replace(project["name"], "") for r in tmp_res] + tmp_res = re.findall( + rf"{project['name']}\![0-9]{{1,5}}", + warn, + ) + failed_MRs += [ + r.replace(project["name"], "") for r in tmp_res + ] df.loc[len(df.index)] = [ project["name"], ",".join(failed_MRs), @@ -221,7 +230,9 @@ def check_status( break else: msgs[slot][date_back]["build_id"] = tmp_build_id - msgs[slot][date_back]["df"] = df.reset_index(drop=True) + msgs[slot][date_back]["df"] = df.reset_index( + drop=True + ) break except AttributeError as err: logging.warning( @@ -230,34 +241,40 @@ def check_status( ) stream = "" for slot, m_values in msgs.items(): - sorted_m_values = dict(sorted(m_values.items(), key=lambda x: x[0])) + sorted_m_values = dict( + sorted(m_values.items(), key=lambda x: x[0]) + ) if html: stream += f"

{slot}

\n" for date_back, values in sorted_m_values.items(): parsed_date = date_back.strftime(self.date_format) if values: if html: - stream += f"
{parsed_date}/{values['build_id']}" - stream += f"link to " + stream += f"
{parsed_date}/{values['build_id']}" # noqa: E501 + stream += f"link to " # noqa: E501 stream += f"{slot}/{values['build_id']}
" pretty_df = values["df"].style.applymap(color_values) stream += f"{pretty_df.to_html()}
" else: - stream += f"-> {slot}/{parsed_date}/{values['build_id']}:\n" + stream += ( + f"-> {slot}/{parsed_date}/{values['build_id']}:\n" + ) table = tabulate( values["df"], - headers=list(map("\n".join, values["df"].columns.tolist())), + headers=list( + map("\n".join, values["df"].columns.tolist()) + ), tablefmt="pretty", ) stream += f"{table}\n" else: if html: - stream += ( - f"
{parsed_date}/(No build)" - ) + stream += f"
{parsed_date}/(No build)" # noqa: E501 stream += "No build available for this day.
" else: - stream += f"-> {slot}/{parsed_date}: No slot available\n" + stream += ( + f"-> {slot}/{parsed_date}: No slot available\n" + ) if filepath: with open(filepath, "w") as f: f.write(stream) diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_status_checker.py b/test/test_status_checker.py new file mode 100644 index 0000000..6e0b95d --- /dev/null +++ b/test/test_status_checker.py @@ -0,0 +1,17 @@ +import subprocess + + +def test_cli_simple_training(): + ex_generate = subprocess.run( + [ + "python", + "run.py", + "current-status", + "--slots", + "lhcb-sim11", + ], + check=True, + capture_output=True, + text=True, + ) + assert ex_generate.returncode == 0