From b8153b2eecde510009746ea0ef6a69f81d3ae36d Mon Sep 17 00:00:00 2001 From: Jan Stourac Date: Thu, 2 Nov 2023 00:45:27 +0100 Subject: [PATCH] Add a GitHub action to perform validation of the `params.env` file --- .github/workflows/params-env.yaml | 23 +++ ci/check-params-env.sh | 234 ++++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 .github/workflows/params-env.yaml create mode 100755 ci/check-params-env.sh diff --git a/.github/workflows/params-env.yaml b/.github/workflows/params-env.yaml new file mode 100644 index 000000000..7dcffdf1b --- /dev/null +++ b/.github/workflows/params-env.yaml @@ -0,0 +1,23 @@ +--- +name: Validation of params.env content (image SHAs) +on: # yamllint disable-line rule:truthy + pull_request: + paths: + - 'manifests/base/params.env' + +permissions: + contents: read + +jobs: + validation-of-params-env: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install dependencies + run: | + sudo apt-get install -y skopeo jq + + - name: Validate the 'manifests/base/params.env' file content + run: | + bash ./ci/check-params-env.sh diff --git a/ci/check-params-env.sh b/ci/check-params-env.sh new file mode 100755 index 000000000..4e8a19ee8 --- /dev/null +++ b/ci/check-params-env.sh @@ -0,0 +1,234 @@ +#!/bin/bash +# +# This script serves to check and validate the `params.env` file that contains +# definitions of the notebook images that are supposed to be used in the resulting +# release. +# +# It is verified that particular image link exists and is a proper type for the +# assigned variable name. Structure of the file is also checked. +# +# THIS FILE DOESN'T CHECK THAT THE USED LINK TO IMAGE IS THE LATEST ONE AVAILABLE! +# +# This script uses `skopeo` and `jq` tools installed locally for retrieving +# information about the particular remote images. +# +# Local execution: ./ci/check-params-env.sh +# Note: please execute from the root directory so that relative path matches +# +# In case of the PR on GitHub, this check is tied to GitHub actions automatically, +# see `.github/workflows` directory. + +PARAMS_ENV_PATH="manifests/base/params.env" +EXPECTED_NUM_RECORDS=18 + +function check_variables_uniq() { + local params_env_path="${1}" + + echo "Checking that all variables in the file '${params_env_path}' are unique" + + local content + content=$(sed 's#\(.*\)=.*#\1#' "${params_env_path}" | sort) + + local num_records + num_records=$(echo "${content}" | wc -l) + + local num_uniq_records + num_uniq_records=$(echo "${content}" | uniq | wc -l) + + test "${num_records}" -eq "${num_uniq_records}" || { + echo "Some of the records in the file aren't unique!" + return 1 + } + + test "${num_records}" -eq "${EXPECTED_NUM_RECORDS}" || { + echo "Number of records in the file is incorrect - expected '${EXPECTED_NUM_RECORDS}' but got '${num_records}'!" + return 1 + } + + echo "---------------------------------------------" +} + +function check_image_variable_matches_name_and_commitref() { + local image_variable="${1}" + local image_name="${2}" + local image_commitref="${3}" + + local expected_name + local expected_commitref + case "${image_variable}" in + odh-minimal-notebook-image-n) + expected_name="odh-notebook-jupyter-minimal-ubi9-python-3.9" + expected_commitref="2023b" + ;; + odh-minimal-notebook-image-n-1) + expected_name="odh-notebook-jupyter-minimal-ubi9-python-3.9" + expected_commitref="2023a" + ;; + odh-minimal-notebook-image-n-2) + expected_name="odh-notebook-jupyter-minimal-ubi8-python-3.8" + expected_commitref="main" + ;; + odh-minimal-gpu-notebook-image-n) + expected_name="odh-notebook-jupyter-minimal-ubi9-python-3.9" + expected_commitref="2023b" + ;; + odh-minimal-gpu-notebook-image-n-1) + expected_name="odh-notebook-jupyter-minimal-ubi9-python-3.9" + expected_commitref="2023a" + ;; + odh-minimal-gpu-notebook-image-n-2) + expected_name="odh-notebook-jupyter-minimal-ubi8-python-3.8" + expected_commitref="main" + ;; + odh-pytorch-gpu-notebook-image-n) + expected_name="odh-notebook-jupyter-pytorch-ubi9-python-3.9" + expected_commitref="2023b" + ;; + odh-pytorch-gpu-notebook-image-n-1) + expected_name="odh-notebook-jupyter-pytorch-ubi9-python-3.9" + expected_commitref="2023a" + ;; + odh-pytorch-gpu-notebook-image-n-2) + expected_name="odh-notebook-cuda-jupyter-pytorch-ubi8-python-3.8" + expected_commitref="main" + ;; + odh-generic-data-science-notebook-image-n) + expected_name="odh-notebook-jupyter-datascience-ubi9-python-3.9" + expected_commitref="2023b" + ;; + odh-generic-data-science-notebook-image-n-1) + expected_name="odh-notebook-jupyter-datascience-ubi9-python-3.9" + expected_commitref="2023a" + ;; + odh-generic-data-science-notebook-image-n-2) + expected_name="odh-notebook-jupyter-datascience-ubi8-python-3.8" + expected_commitref="main" + ;; + odh-tensorflow-gpu-notebook-image-n) + expected_name="odh-notebook-cuda-jupyter-tensorflow-ubi9-python-3.9" + expected_commitref="2023b" + ;; + odh-tensorflow-gpu-notebook-image-n-1) + expected_name="odh-notebook-cuda-jupyter-tensorflow-ubi9-python-3.9" + expected_commitref="2023a" + ;; + odh-tensorflow-gpu-notebook-image-n-2) + expected_name="odh-notebook-cuda-jupyter-tensorflow-ubi8-python-3.8" + expected_commitref="main" + ;; + odh-trustyai-notebook-image-n) + expected_name="odh-notebook-jupyter-trustyai-ubi9-python-3.9" + expected_commitref="2023b" + ;; + odh-trustyai-notebook-image-n-1) + expected_name="odh-notebook-jupyter-trustyai-ubi9-python-3.9" + expected_commitref="2023a" + ;; + odh-habana-notebook-image-n) + expected_name="odh-notebook-habana-jupyter-1.10.0-ubi8-python-3.8" + # expected_commitref="2023b" + expected_commitref="main" + ;; + *) + echo "Unimplemented variable name: '${image_variable}'" + return 1 + esac + + test "${image_name}" = "${expected_name}" || { + echo "Image URL for points to an incorrect image: expected name '${expected_name}'; actual '${image_name}'" + return 1 + } + + test "${image_commitref}" = "${expected_commitref}" || { + echo "Image URL for points to an incorrect image: expected commitref '${expected_commitref}'; actual '${image_commitref}'" + return 1 + } +} + +function check_image() { + local image_variable="${1}" + local image_url="${2}" + + echo "Checking metadata for image '${image_variable}' with URL '${image_url}'" + + local image_metadata + local image_name + local image_commitref + + image_metadata="$(skopeo inspect --config "docker://${image_url}")" || return 1 + image_name=$(echo "${image_metadata}" | jq --raw-output '.config.Labels.name') || return 1 + image_commitref=$(echo "${image_metadata}" | jq --raw-output '.config.Labels."io.openshift.build.commit.ref"') || return 1 + + test -n "${image_name}" || { + echo "Couldn't retrieve the name of the image - got empty value!" + return 1 + } + + echo "Image name retrieved: '${image_name}'" + + check_image_variable_matches_name_and_commitref "${image_variable}" "${image_name}" "${image_commitref}" || return 1 + + echo "---------------------------------------------" +} + +ret_code=0 + +echo "Starting check for file: '${PARAMS_ENV_PATH}'" +echo "---------------------------------------------" + +check_variables_uniq "${PARAMS_ENV_PATH}" || { + echo "ERROR: Variable names in the file failed validation!" + echo "----------------------------------------------------" + ret_code=1 +} + +while IFS= read -r LINE; do +# IFS=$'\n' +# for LINE in $(cat "${PARAMS_ENV_PATH}"); do + echo "Checking format of: '${LINE}'" + [[ "${LINE}" = *[[:space:]]* ]] && { + echo "ERROR: Line contains white-space and it shouldn't!" + echo "--------------------------------------------------" + ret_code=1 + continue + } + [[ "${LINE}" != *=* ]] && { + echo "ERROR: Line doesn't contain '=' and it should!" + echo "----------------------------------------------" + ret_code=1 + continue + } + + IMAGE_VARIABLE=$(echo "${LINE}" | cut --delimiter '=' --field 1) + IMAGE_URL=$(echo "${LINE}" | cut --delimiter '=' --field 2) + + test -n "${IMAGE_VARIABLE}" || { + echo "ERROR: Couldn't parse image variable - got empty value!" + echo "-------------------------------------------------------" + ret_code=1 + continue + } + + test -n "${IMAGE_URL}" || { + echo "ERROR: Couldn't parse image URL - got empty value!" + echo "--------------------------------------------------" + ret_code=1 + continue + } + + check_image "${IMAGE_VARIABLE}" "${IMAGE_URL}" || { + echo "ERROR: Image isn't okay!" + echo "------------------------" + ret_code=1 + continue + } +done < "${PARAMS_ENV_PATH}" + +echo "" +if test "${ret_code}" -eq 0; then + echo "Validation of '${PARAMS_ENV_PATH}' was successful! Congrats :)" +else + echo "The '${PARAMS_ENV_PATH}' file isn't valid, please check above!" +fi + +exit "${ret_code}" \ No newline at end of file