From 20248b3ab357706843d0a6cf3a199d94abf6ea14 Mon Sep 17 00:00:00 2001 From: OpenShift Helm Charts Bot <83200018+openshift-helm-charts-bot@users.noreply.github.com> Date: Fri, 19 Jul 2024 12:43:16 -0500 Subject: [PATCH] Release-1.7.3 (#1552) Co-authored-by: openshift-helm-charts-bot <41898282+github-actions[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 5 + .github/workflows/test-cluster-access.yml | 5 + scripts/requirements.txt | 63 ++++---- scripts/src/chartprreview/chartprreview.py | 2 +- scripts/src/checkprcontent/checkpr.py | 2 +- scripts/src/precheck/submission.py | 78 +++++++++- scripts/src/precheck/submission_test.py | 134 ++++++++++++++++++ scripts/src/release/releasechecker.py | 4 +- scripts/src/saforcertadmin/create_sa.sh | 3 +- scripts/src/saforcertadmin/token_secret.yaml | 8 ++ .../saforcharttesting/saforcharttesting.py | 20 +++ 11 files changed, 285 insertions(+), 39 deletions(-) create mode 100644 scripts/src/saforcertadmin/token_secret.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d9cda1ae76..33dfd47a79 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,6 +4,11 @@ on: pull_request_target: types: [opened, synchronize, reopened, edited, ready_for_review, labeled] +env: + # Temporary workaround. See + # https://github.com/redhat-actions/openshift-tools-installer/issues/105 + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + jobs: setup: name: Setup CI diff --git a/.github/workflows/test-cluster-access.yml b/.github/workflows/test-cluster-access.yml index 3d2b2598db..c83e234e57 100644 --- a/.github/workflows/test-cluster-access.yml +++ b/.github/workflows/test-cluster-access.yml @@ -6,6 +6,11 @@ name: Test Cluster Access on: workflow_dispatch: +env: + # Temporary workaround. See + # https://github.com/redhat-actions/openshift-tools-installer/issues/105 + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + jobs: test-cluster-access: name: Test Cluster Access diff --git a/scripts/requirements.txt b/scripts/requirements.txt index ef110623c0..8e7b8f29e9 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,38 +1,35 @@ -attrs==21.2.0 -certifi==2020.12.5 -chardet==4.0.0 -docker==6.1.3 -environs==9.5.0 -execnet==1.9.0 -gitdb==4.0.7 -GitPython==3.1.18 +attrs==23.2.0 +certifi==2024.6.2 +chardet==5.2.0 +docker==7.1.0 +environs==11.0.0 +execnet==2.1.1 +gitdb==4.0.11 +GitPython==3.1.43 glob2==0.7 -idna==2.10 -iniconfig==1.1.1 -mako==1.2.3 -MarkupSafe==2.0.1 -packaging==21.0 -parse==1.19.0 -parse-type==0.5.2 -pluggy==0.13.1 -psutil==5.8.0 -py==1.10.0 -PyGithub==1.55 -pyparsing==2.4.7 -pytest==6.2.4 -pytest-bdd==4.1.0 -pytest-forked==1.3.0 -pytest-xdist==2.4.0 +idna==3.7 +iniconfig==2.0.0 +mako==1.3.5 +MarkupSafe==2.1.5 +packaging==24.1 +parse==1.20.2 +parse-type==0.6.2 +pluggy==1.5.0 +psutil==5.9.8 +PyGithub==2.3.0 +pyparsing==3.1.2 +pytest==8.2.2 +pytest-bdd==7.2.0 PyYAML==6.0.1 -requests==2.26.0 -responses==0.23.1 -retrying==1.3.3 -semantic-version==2.8.5 -semver==2.13.0 +requests==2.32.3 +responses==0.25.3 +retrying==1.3.4 +semantic-version==2.10.0 +semver==3.0.2 six==1.16.0 -smmap==4.0.0 +smmap==5.0.1 toml==0.10.2 -urllib3==1.26.5 -websocket-client==1.2.1 -analytics-python==1.4.0 +urllib3==2.2.2 +websocket-client==1.8.0 +analytics-python==1.4.post1 behave==1.2.6 diff --git a/scripts/src/chartprreview/chartprreview.py b/scripts/src/chartprreview/chartprreview.py index 7cf0e28c19..33e0099118 100644 --- a/scripts/src/chartprreview/chartprreview.py +++ b/scripts/src/chartprreview/chartprreview.py @@ -446,7 +446,7 @@ def check_report_success(directory, api_url, report_path, report_info_path, vers if "charts.openshift.io/certifiedOpenShiftVersions" in annotations: full_version = annotations["charts.openshift.io/certifiedOpenShiftVersions"] - if not semver.VersionInfo.isvalid(full_version): + if not semver.VersionInfo.is_valid(full_version): msg = f"[ERROR] certified OpenShift version not conforming to SemVer spec: {full_version}" write_error_log(directory, msg) sys.exit(1) diff --git a/scripts/src/checkprcontent/checkpr.py b/scripts/src/checkprcontent/checkpr.py index 29932af7d8..8190a15a71 100644 --- a/scripts/src/checkprcontent/checkpr.py +++ b/scripts/src/checkprcontent/checkpr.py @@ -200,7 +200,7 @@ def ensure_only_chart_is_modified(api_url, repository, branch): gitutils.add_output("organization", organization) gitutils.add_output("chart-name", chart) - if not semver.VersionInfo.isvalid(version): + if not semver.VersionInfo.is_valid(version): msg = ( f"[ERROR] Helm chart version is not a valid semantic version: {version}" ) diff --git a/scripts/src/precheck/submission.py b/scripts/src/precheck/submission.py index 2ed6175aaf..6a7043cf50 100644 --- a/scripts/src/precheck/submission.py +++ b/scripts/src/precheck/submission.py @@ -1,6 +1,13 @@ import os import re +import requests import semver +import yaml + +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader from dataclasses import dataclass, field @@ -36,6 +43,14 @@ class WebCatalogOnlyError(SubmissionError): pass +class HelmIndexError(SubmissionError): + pass + + +class ReleaseTagError(SubmissionError): + pass + + @dataclass class Chart: """Represents a Helm Chart @@ -59,7 +74,7 @@ def register_chart_info(self, category, organization, name, version): msg = "[ERROR] A PR must contain only one chart. Current PR includes files for multiple charts." raise DuplicateChartError(msg) - if not semver.VersionInfo.isvalid(version): + if not semver.VersionInfo.is_valid(version): msg = ( f"[ERROR] Helm chart version is not a valid semantic version: {version}" ) @@ -73,6 +88,53 @@ def register_chart_info(self, category, organization, name, version): def get_owners_path(self): return f"charts/{self.category}/{self.organization}/{self.name}/OWNERS" + def get_release_tag(self): + return f"{self.organization}-{self.name}-{self.version}" + + def check_index(self, index): + """Check if the chart is present in the Helm index + + Args: + index (dict): Content of the Helm repo index + + Raise: + HelmIndexError if: + * The provided index is malformed + * The Chart is already present in the index + + """ + try: + chart_entry = index["entries"].get(self.name, []) + except KeyError as e: + raise HelmIndexError(f"Malformed index {index}") from e + + for chart in chart_entry: + if chart["version"] == self.version: + msg = f"[ERROR] Helm chart release already exists in the index.yaml: {self.version}" + raise HelmIndexError(msg) + + def check_release_tag(self, repository: str): + """Check for the existence of the chart's release tag on the provided repository. + + Args: + repository (str): Name of the GitHub repository to check for existing tag. + (e.g. "openshift-helm-charts/charts") + + Raise: ReleaseTagError if the tag already exists in the GitHub repo. + + """ + tag_name = self.get_release_tag() + tag_api = f"https://api.github.com/repos/{repository}/git/ref/tags/{tag_name}" + headers = { + "Accept": "application/vnd.github.v3+json", + "Authorization": f'Bearer {os.environ.get("BOT_TOKEN")}', + } + print(f"[INFO] checking tag: {tag_api}") + r = requests.head(tag_api, headers=headers) + if r.status_code == 200: + msg = f"[ERROR] Helm chart release already exists in the GitHub Release/Tag: {tag_name}" + raise ReleaseTagError(msg) + @dataclass class Report: @@ -439,3 +501,17 @@ def get_file_type(file_path): return "owners", owners_match return "unknwown", None + + +def download_index_data(repository, branch="gh_pages"): + """Download the helm repository index""" + r = requests.get( + f"https://raw.githubusercontent.com/{repository}/{branch}/index.yaml" + ) + + if r.status_code == 200: + data = yaml.load(r.text, Loader=Loader) + else: + data = {} + + return data diff --git a/scripts/src/precheck/submission_test.py b/scripts/src/precheck/submission_test.py index 161b048063..b4be69c75f 100644 --- a/scripts/src/precheck/submission_test.py +++ b/scripts/src/precheck/submission_test.py @@ -618,3 +618,137 @@ def test_is_valid_web_catalog_only(test_scenario): test_scenario.input_submission.is_valid_web_catalog_only(repo_path=temp_dir) == test_scenario.expected_output ) + + +def create_new_index(charts: list[submission.Chart] = []): + """Create the JSON representation of a Helm chart index containing the provided list of charts + + The resulting index only contains the required information for the check_index to work. + + """ + index = {"apiVersion": "v1", "entries": {}} + + for chart in charts: + chart_entries = index["entries"].get(chart.name, []) + chart_entries.append( + { + "name": f"{chart.name}", + "version": f"{chart.version}", + } + ) + + index["entries"][chart.name] = chart_entries + + return index + + +@dataclass +class CheckIndexScenario: + chart: submission.Chart = field( + default_factory=lambda: submission.Chart( + category=expected_category, + organization=expected_organization, + name=expected_name, + version=expected_version, + ) + ) + index: dict = field(default_factory=lambda: create_new_index()) + excepted_exception: contextlib.ContextDecorator = field( + default_factory=lambda: contextlib.nullcontext() + ) + + +scenarios_check_index = [ + # Chart is not present in the index + CheckIndexScenario( + index=create_new_index([submission.Chart(name="not-awesome", version="0.42")]) + ), + # Chart is present but does not contain submitted version + CheckIndexScenario( + index=create_new_index([submission.Chart(name=expected_name, version="0.42")]) + ), + # Submitted version is present in index + CheckIndexScenario( + index=create_new_index( + [submission.Chart(name=expected_name, version=expected_version)] + ), + excepted_exception=pytest.raises( + submission.HelmIndexError, + match="Helm chart release already exists in the index.yaml", + ), + ), + # Index is empty + CheckIndexScenario(), + # Index is an empty dict + CheckIndexScenario( + index={}, + excepted_exception=pytest.raises( + submission.HelmIndexError, match="Malformed index" + ), + ), +] + + +@pytest.mark.parametrize("test_scenario", scenarios_check_index) +def test_check_index(test_scenario): + with test_scenario.excepted_exception: + test_scenario.chart.check_index(test_scenario.index) + + +@dataclass +class CheckReleaseTagScenario: + chart: submission.Chart = field( + default_factory=lambda: submission.Chart( + category=expected_category, + organization=expected_organization, + name=expected_name, + version=expected_version, + ) + ) + exising_tags: list[str] = field(default_factory=lambda: list()) + excepted_exception: contextlib.ContextDecorator = field( + default_factory=lambda: contextlib.nullcontext() + ) + + +scenarios_check_release_tag = [ + # A release doesn't exist for this org + CheckReleaseTagScenario(exising_tags=["notacme-notawesome-0.42"]), + # A release exist for this org, but not for this chart + CheckReleaseTagScenario(exising_tags=[f"{expected_organization}-notawesome-0.42"]), + # A release exist for this Chart but not in this version + CheckReleaseTagScenario( + exising_tags=[f"{expected_organization}-{expected_name}-0.42"], + ), + # A release exist for this Chart in this version + CheckReleaseTagScenario( + exising_tags=[f"{expected_organization}-{expected_name}-{expected_version}"], + excepted_exception=pytest.raises( + submission.ReleaseTagError, + match="Helm chart release already exists in the GitHub Release/Tag", + ), + ), +] + + +@pytest.mark.parametrize("test_scenario", scenarios_check_release_tag) +@responses.activate +def test_check_release_tag(test_scenario): + chart_release_tag = test_scenario.chart.get_release_tag() + + if chart_release_tag not in test_scenario.exising_tags: + responses.head( + f"https://api.github.com/repos/my-fake-org/my-fake-repo/git/ref/tags/{chart_release_tag}", + # json=[{"filename": file} for file in test_scenario.modified_files], + status=404, + ) + + for tag in test_scenario.exising_tags: + # Mock GitHub API + responses.head( + f"https://api.github.com/repos/my-fake-org/my-fake-repo/git/ref/tags/{tag}", + # json=[{"filename": file} for file in test_scenario.modified_files], + ) + + with test_scenario.excepted_exception: + test_scenario.chart.check_release_tag(repository="my-fake-org/my-fake-repo") diff --git a/scripts/src/release/releasechecker.py b/scripts/src/release/releasechecker.py index c8647583cc..9f9d0124ae 100644 --- a/scripts/src/release/releasechecker.py +++ b/scripts/src/release/releasechecker.py @@ -104,7 +104,7 @@ def check_if_dev_release_branch(sender, pr_branch, pr_body, api_url, pr_head_rep return False version = pr_branch.removeprefix(releaser.DEV_PR_BRANCH_NAME_PREFIX) - if not semver.VersionInfo.isvalid(version): + if not semver.VersionInfo.is_valid(version): print( f"Release part ({version}) of branch name {pr_branch} is not a valid semantic version." ) @@ -139,7 +139,7 @@ def check_if_charts_release_branch(sender, pr_branch, pr_body, api_url, pr_head_ return False version = pr_branch.removeprefix(releaser.CHARTS_PR_BRANCH_NAME_PREFIX) - if not semver.VersionInfo.isvalid(version): + if not semver.VersionInfo.is_valid(version): print( f"Release part ({version}) of branch name {pr_branch} is not a valid semantic version." ) diff --git a/scripts/src/saforcertadmin/create_sa.sh b/scripts/src/saforcertadmin/create_sa.sh index dc59c7f2a2..5c17e70258 100755 --- a/scripts/src/saforcertadmin/create_sa.sh +++ b/scripts/src/saforcertadmin/create_sa.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash user_name='rh-cert-user' +token_secret='rh-cert-user-token' oc create sa $user_name -token_secret=$(oc get secrets --field-selector=type=kubernetes.io/service-account-token -o=jsonpath="{.items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=='"$user_name"')].metadata.name}") +oc apply -f token_secret.yaml token=$(oc get secret $token_secret -o json | jq -r .data.token | base64 -d) oc apply -f cluster_role_binding.yaml diff --git a/scripts/src/saforcertadmin/token_secret.yaml b/scripts/src/saforcertadmin/token_secret.yaml new file mode 100644 index 0000000000..ed94088dec --- /dev/null +++ b/scripts/src/saforcertadmin/token_secret.yaml @@ -0,0 +1,8 @@ +# Creates the secret. Cluster will populate with data. +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: rh-cert-user-token + annotations: + kubernetes.io/service-account.name: rh-cert-user diff --git a/scripts/src/saforcharttesting/saforcharttesting.py b/scripts/src/saforcharttesting/saforcharttesting.py index aab88d0f35..431ec20b80 100644 --- a/scripts/src/saforcharttesting/saforcharttesting.py +++ b/scripts/src/saforcharttesting/saforcharttesting.py @@ -24,6 +24,17 @@ namespace: ${name} """ +token_template = """\ +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: token-${name} + namespace: ${name} + annotations: + kubernetes.io/service-account.name: ${name} +""" + role_template = """\ kind: Role apiVersion: rbac.authorization.k8s.io/v1 @@ -164,6 +175,14 @@ def create_serviceaccount(namespace): print("[ERROR] creating ServiceAccount:", stderr) +def create_tokensecret(namespace): + print("creating token Secret:", namespace) + stdout, stderr = apply_config(token_template, name=namespace) + print("stdout:\n", stdout, sep="") + if stderr.strip(): + print("[ERROR] creating token Secret:", stderr) + + def create_role(namespace): print("creating Role:", namespace) stdout, stderr = apply_config(role_template, name=namespace) @@ -344,6 +363,7 @@ def main(): if args.create: create_namespace(args.create) create_serviceaccount(args.create) + create_tokensecret(args.create) create_role(args.create) create_rolebinding(args.create) create_clusterrole(args.create)