Update digest updater workflow

opendatahub-io · Mar 21, 2024 · 700c017 · 700c017
1 parent 503090e
commit 700c017
Showing 1 changed file with 152 additions and 39 deletions.
diff --git a/.github/workflows/notebooks-digest-updater-upstream.yaml b/.github/workflows/notebooks-digest-updater-upstream.yaml
@@ -40,6 +40,73 @@ jobs:
          git checkout -b ${{ env.DIGEST_UPDATER_BRANCH }}
          git push --set-upstream origin ${{ env.DIGEST_UPDATER_BRANCH }}
 
+  update-runtimes:
+      needs: [initialize]
+      runs-on: ubuntu-latest
+      permissions:
+        contents: write
+      steps:
+        - name: Configure Git
+          run: |
+            git config --global user.email "github-actions[bot]@users.noreply.github.com"
+            git config --global user.name "GitHub Actions"
+
+        # Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name
+        - name: Retrive latest commit hash from the release branch
+          id: hash-n
+          shell: bash
+          run: |
+            PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N&per_page=1)
+            echo "HASH_N=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT}
+
+        # Checkout the release branch to apply the updates
+        - name: Checkout release branch
+          uses: actions/checkout@v4
+          with:
+            ref: ${{ env.DIGEST_UPDATER_BRANCH }}
+
+        - name: Update Runtimes
+          run: |
+              echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}}
+              PATHS=("jupyter/datascience/anaconda-python-3.8/runtime-images/datascience-ubi8-py38.json"
+              "jupyter/datascience/anaconda-python-3.8/runtime-images/pytorch-ubi8-py38.json"
+              "jupyter/datascience/anaconda-python-3.8/runtime-images/tensorflow-ubi8-py38.json"
+              "jupyter/datascience/anaconda-python-3.8/runtime-images/ubi8-py38.json"
+              "jupyter/datascience/ubi8-python-3.8/runtime-images/datascience-ubi8-py38.json"
+              "jupyter/datascience/ubi8-python-3.8/runtime-images/pytorch-ubi8-py38.json"
+              "jupyter/datascience/ubi8-python-3.8/runtime-images/tensorflow-ubi8-py38.json"
+              "jupyter/datascience/ubi8-python-3.8/runtime-images/ubi8-py38.json"
+              "jupyter/datascience/ubi9-python-3.9/runtime-images/datascience-ubi9-py39.json"
+              "jupyter/datascience/ubi9-python-3.9/runtime-images/pytorch-ubi9-py39.json"
+              "jupyter/datascience/ubi9-python-3.9/runtime-images/tensorflow-ubi9-py39.json"
+              "jupyter/datascience/ubi9-python-3.9/runtime-images/ubi9-py39.json")
+
+              for ((i=0;i<${#PATHS[@]};++i)); do
+                path=${PATHS[$i]}
+                img=$(cat ${path} | jq -r '.metadata.image_name')
+                name=$(echo "$path" | sed 's#.*runtime-images/\(.*\)-py.*#\1#')
+                py_version=$(echo "$path" | grep -o 'python-[0-9]\.[0-9]')
+                # Handling specific cases
+                if [[ $name == *tensorflow* ]]; then
+                    name="cuda-$name"
+                elif [[ $name == ubi* ]]; then
+                    name="minimal-$name"
+                fi
+                registry=$(echo $img | cut -d '@' -f1)
+                regex="runtime-$name-$py_version-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}"
+                echo "CHECKING: "  $regex
+                latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
+                digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
+                output=$registry@$digest
+                echo "NEW: " $output
+                jq --arg output "$output" '.metadata.image_name = $output' "$path" > "$path.tmp" && mv "$path.tmp" "$path"
+              done
+              if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
+                git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add jupyter/datascience/* && git commit -m "Update file via  ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+              else
+                echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
+              fi
+
   update-n-version:
     needs: [initialize]
     runs-on: ubuntu-latest
@@ -51,53 +118,75 @@ jobs:
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git config --global user.name "GitHub Actions"
 
-      # Get the latest weekly build commit hash: https://github.com/opendatahub-io/notebooks/commits/2024a
-      - name: Checkout upstream notebooks repo
-        uses: actions/checkout@v4
-        with:
-          repository: opendatahub-io/notebooks.git
-          ref: ${{ env.RELEASE_VERSION_N }}
-
-      - name: Retrieve latest weekly commit hash from the release branch
+      # Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name
+      - name: Retrive latest commit hash from the release branch
         id: hash-n
         shell: bash
         run: |
-          echo "HASH_N=$(git rev-parse --short HEAD)" >> ${GITHUB_OUTPUT}
+          PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N&per_page=1)
+          echo "HASH_N=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT}
 
       # Checkout the release branch to apply the updates
       - name: Checkout release branch
         uses: actions/checkout@v4
         with:
           ref: ${{ env.DIGEST_UPDATER_BRANCH }}
 
-      - name: Fetch digest, and update the params.env file
+      - name: Update the param.env file
         run: |
               echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}}
-              IMAGES=("odh-minimal-notebook-image-n" "odh-minimal-gpu-notebook-image-n" "odh-pytorch-gpu-notebook-image-n" "odh-generic-data-science-notebook-image-n" "odh-tensorflow-gpu-notebook-image-n" "odh-trustyai-notebook-image-n")
+              IMAGES=("odh-minimal-notebook-image-n"
+               "odh-minimal-gpu-notebook-image-n"
+               "odh-pytorch-gpu-notebook-image-n"
+               "odh-generic-data-science-notebook-image-n"
+               "odh-tensorflow-gpu-notebook-image-n"
+               "odh-trustyai-notebook-image-n"
+               "odh-codeserver-notebook-n"
+               "odh-rstudio-notebook-n"
+               "odh-rstudio-gpu-notebook-n")
+
               for ((i=0;i<${#IMAGES[@]};++i)); do
                 image=${IMAGES[$i]}
-                echo $image
+                echo "CHECKING: " $image
                 img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2)
                 registry=$(echo $img | cut -d '@' -f1)
                 src_tag=$(skopeo inspect docker://$img | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
                 regex="$src_tag-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}"
                 latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
                 digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
                 output=$registry@$digest
-                echo $output
+                echo "NEW:" $output
                 sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env
               done
-              git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+              if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
+                git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+              else
+                echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
+              fi
 
-      - name: Fetch digest, and update the commit.env file
+      - name: Update the commit.env file
         run: |
             echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}}
-            COMMIT=("odh-minimal-notebook-image-commit-n" "odh-minimal-gpu-notebook-image-commit-n" "odh-pytorch-gpu-notebook-image-commit-n" "odh-generic-data-science-notebook-image-commit-n" "odh-tensorflow-gpu-notebook-image-commit-n" "odh-trustyai-notebook-image-commit-n")
+            COMMIT=("odh-minimal-notebook-image-commit-n"
+             "odh-minimal-gpu-notebook-image-commit-n"
+             "odh-pytorch-gpu-notebook-image-commit-n"
+             "odh-generic-data-science-notebook-image-commit-n"
+             "odh-tensorflow-gpu-notebook-image-commit-n"
+             "odh-trustyai-notebook-image-commit-n"
+             "odh-codeserver-notebook-n"
+             "odh-rstudio-notebook-n"
+             "odh-rstudio-gpu-notebook-n")
+
             for val in "${COMMIT[@]}"; do
               echo $val
               sed -i "s|${val}=.*|${val}=${{ steps.hash-n.outputs.HASH_N }}|" manifests/base/commit.env
             done
-            git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+            if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
+              git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+            else
+              echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
+            fi
+
   update-n-1-version:
     needs: [initialize, update-n-version]
     runs-on: ubuntu-latest
@@ -109,56 +198,77 @@ jobs:
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git config --global user.name "GitHub Actions"
 
-      # Get the latest weekly build commit hash: https://github.com/opendatahub-io/notebooks/commits/2023b
-      - name: Checkout upstream notebooks repo
-        uses: actions/checkout@v4
-        with:
-          repository: opendatahub-io/notebooks.git
-          ref: ${{ env.RELEASE_VERSION_N_1 }}
-
-      - name: Retrieve latest weekly commit hash from the release branch
+      # Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name
+      - name: Retrive latest commit hash from the release branch
         id: hash-n-1
         shell: bash
         run: |
-          echo "HASH_N_1=$(git rev-parse --short HEAD)" >> ${GITHUB_OUTPUT}
+          PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N_1&per_page=1)
+          echo "HASH_N_1=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT}
 
       # Checkout the release branch to apply the updates
       - name: Checkout release branch
         uses: actions/checkout@v4
         with:
           ref: ${{ env.DIGEST_UPDATER_BRANCH }}
 
-      - name: Fetch digest, and update the params.env file
+      - name: Update the param.env file
         run: |
               echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}}
-              IMAGES=("odh-minimal-notebook-image-n-1" "odh-minimal-gpu-notebook-image-n-1" "odh-pytorch-gpu-notebook-image-n-1" "odh-generic-data-science-notebook-image-n-1" "odh-tensorflow-gpu-notebook-image-n-1" "odh-trustyai-notebook-image-n-1")
+              IMAGES=("odh-minimal-notebook-image-n-1"
+               "odh-minimal-gpu-notebook-image-n-1"
+               "odh-pytorch-gpu-notebook-image-n-1"
+               "odh-generic-data-science-notebook-image-n-1"
+               "odh-tensorflow-gpu-notebook-image-n-1"
+               "odh-trustyai-notebook-image-n-1"
+               "odh-codeserver-notebook-n-1"
+               "odh-rstudio-notebook-n-1"
+               "odh-rstudio-gpu-notebook-n-1")
+
               for ((i=0;i<${#IMAGES[@]};++i)); do
                 image=${IMAGES[$i]}
-                echo $image
+                echo "CHECKING: " $image
                 img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2)
                 registry=$(echo $img | cut -d '@' -f1)
                 src_tag=$(skopeo inspect docker://$img | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
                 regex="$src_tag-${{ env.RELEASE_VERSION_N_1}}-\d+-${{ steps.hash-n-1.outputs.HASH_N_1 }}"
                 latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
                 digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
                 output=$registry@$digest
-                echo $output
+                echo "NEW:" $output
                 sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env
               done
-              git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
-
-      - name: Fetch digest, and update the commit.env file
+              if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
+                git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+              else
+                  echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N_1}}"
+              fi
+      - name: Update the commit.env file
         run: |
               echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}}
-              COMMIT=("odh-minimal-notebook-image-commit-n-1" "odh-minimal-gpu-notebook-image-commit-n-1" "odh-pytorch-gpu-notebook-image-commit-n-1" "odh-generic-data-science-notebook-image-commit-n-1" "odh-tensorflow-gpu-notebook-image-commit-n-1" "odh-trustyai-notebook-image-commit-n-1")
+              COMMIT=("odh-minimal-notebook-image-commit-n-1"
+               "odh-minimal-gpu-notebook-image-commit-n-1"
+               "odh-pytorch-gpu-notebook-image-commit-n-1"
+               "odh-generic-data-science-notebook-image-commit-n-1"
+               "odh-tensorflow-gpu-notebook-image-commit-n-1"
+               "odh-trustyai-notebook-image-commit-n-1"
+               "odh-codeserver-notebook-n-1"
+               "odh-rstudio-notebook-n-1"
+               "odh-rstudio-gpu-notebook-n-1")
+
               for val in "${COMMIT[@]}"; do
                 echo $val
                 sed -i "s|${val}=.*|${val}=${{ steps.hash-n-1.outputs.HASH_N_1 }}|" manifests/base/commit.env
               done
-              git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
-  # Creates the Pull Request
+              if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
+                git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+              else
+                echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
+              fi
+
+
   open-pull-request:
-    needs: [update-n-version, update-n-1-version]
+    needs: [update-n-version, update-n-1-version, update-runtimes]
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
@@ -176,8 +286,11 @@ jobs:
           pr_title: "[Digest Updater Action] Update notebook's imageStreams image tag to digest format"
           pr_body: |
             :rocket: This is an automated Pull Request.
-
-            This PR updates the `manifests/base/params.env` file with the latest updated SHA digests of the notebooks (N & N-1).
             Created by `/.github/workflows/notebooks-digest-updater-upstream.yaml`
 
+            This PR updates the following files:
+            - `manifests/base/params.env` file with the latest updated SHA digests of the notebooks (N & N-1).
+            - `manifests/base/commit.env` file with the latest commit (N & N-1).
+            - All the runtime images across `/jupyter/datascience/ubi*-python-*/runtime-images/*.json` paths
+
             :exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.DIGEST_UPDATER_BRANCH }}` branch after merging the changes