Merge remote-tracking branch 'upstream/main' into rhoai-2.13

red-hat-data-services · Aug 27, 2024 · d7e0dc5 · d7e0dc5
2 parents 6104d2b + 5923bd6
commit d7e0dc5
Show file tree

Hide file tree

Showing 4 changed files with 116 additions and 89 deletions.
diff --git a/.github/workflows/notebook-digest-updater.yaml b/.github/workflows/notebook-digest-updater.yaml
@@ -65,61 +65,75 @@ jobs:
         with:
           ref: ${{ env.DIGEST_UPDATER_BRANCH }}
 
-      - name: Fetch digest, and update the params.env file
+      - name: Update the params.env file
         shell: bash
         run: |
           echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}}
-          IMAGES=("odh-minimal-notebook-image-n"
-            "odh-minimal-gpu-notebook-image-n"
-            "odh-pytorch-gpu-notebook-image-n"
-            "odh-generic-data-science-notebook-image-n"
-            "odh-tensorflow-gpu-notebook-image-n"
-            "odh-trustyai-notebook-image-n"
-            "odh-codeserver-notebook-image-n"
-            "odh-habana-notebook-image-n")
-          REGEXES=("v2-${{ env.RELEASE_VERSION_N }}-\d{8}+-${{ steps.hash-n.outputs.HASH_N }}" "cuda-[a-z]+-minimal-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}" "v2-${{ env.RELEASE_VERSION_N }}-\d{8}+-${{ steps.hash-n.outputs.HASH_N }}" \
-                   "v2-${{ env.RELEASE_VERSION_N }}-\d{8}+-${{ steps.hash-n.outputs.HASH_N }}" "cuda-[a-z]+-tensorflow-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}" "v2-${{ env.RELEASE_VERSION_N }}-\d{8}+-${{ steps.hash-n.outputs.HASH_N }}" \
-                   "codeserver-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}"
-                   "habana-[a-z]+-1.13.0-[a-z0-9]+-[a-z]+-3.8-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}")
-
-          for ((i=0;i<${#IMAGES[@]};++i)); do
-            image=${IMAGES[$i]}
-            echo "CHECKING: " $image
+
+          # Get the complete list of images N-version to update
+          PARAMS_ENV_PATH="manifests/base/params.env"
+          IMAGES=$(grep "\-n=" "${PARAMS_ENV_PATH}" | cut -d "=" -f 1)
+
+          # The order of the regexes array should match with the params.env file
+          REGEXES=("v2-${{ env.RELEASE_VERSION_N }}-\d{8}-+${{ steps.hash-n.outputs.HASH_N }}" \
+                  "cuda-[a-z]+-minimal-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}" \
+                  "v2-${{ env.RELEASE_VERSION_N }}-\d{8}-+${{ steps.hash-n.outputs.HASH_N }}" \
+                  "v2-${{ env.RELEASE_VERSION_N }}-\d{8}-+${{ steps.hash-n.outputs.HASH_N }}" \
+                  "cuda-[a-z]+-tensorflow-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}" \
+                  "v2-${{ env.RELEASE_VERSION_N }}-\d{8}-+${{ steps.hash-n.outputs.HASH_N }}" \
+                  "codeserver-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}" \
+                  "habana-[a-z]+-1.13.0-[a-z0-9]+-[a-z]+-3.8-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}" \
+                  "rocm-[a-z]+-minimal-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}" \
+                  "rocm-[a-z]+-pytorch-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}" \
+                  "rocm-[a-z]+-tensorflow-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}")
+
+          i=0
+          for image in ${IMAGES}; do
+            echo "CHECKING: '${image}'"
+            img=$(grep -E "${image}=" "${PARAMS_ENV_PATH}" | cut -d '=' -f2)
+            registry=$(echo "${img}" | cut -d '@' -f1)
+
             regex=${REGEXES[$i]}
-            img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2)
-            registry=$(echo $img | cut -d '@' -f1)
-            latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
-            digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
-            output=$registry@$digest
-            echo "NEW: " $output
-            sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env
+            skopeo_metadata=$(skopeo inspect --retry-times 3 "docker://${img}")
+            latest_tag=$(echo "${skopeo_metadata}" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
+            digest=$(skopeo inspect --retry-times 3 "docker://${registry}:${latest_tag}" | jq .Digest | tr -d '"')
+            output="${registry}@${digest}"
+            echo "NEW: ${output}"
+            sed -i "s|${image}=.*|${image}=${output}|" "${PARAMS_ENV_PATH}"
+            i=$((i+1))
           done
           if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
-            git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
+            git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
+            git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
+            git add "${PARAMS_ENV_PATH}" && \
+            git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
+            git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
           else
             echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
           fi
 
-      - name: Fetch digest, and update the commit.env file
+      - name: Update the commit.env file
         run: |
           echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}}
-          COMMIT=("odh-minimal-notebook-image-commit-n"
-           "odh-minimal-gpu-notebook-image-commit-n"
-           "odh-pytorch-gpu-notebook-image-commit-n"
-           "odh-generic-data-science-notebook-image-commit-n"
-           "odh-tensorflow-gpu-notebook-image-commit-n"
-           "odh-trustyai-notebook-image-commit-n"
-           "odh-codeserver-notebook-image-commit-n"
-           "odh-habana-notebook-image-commit-n")
-
-          for val in "${COMMIT[@]}"; do
-            echo $val
-            sed -i "s|${val}=.*|${val}=${{ steps.hash-n.outputs.HASH_N }}|" manifests/base/commit.env
+
+          COMMIT_ENV_PATH="manifests/base/commit.env"
+
+          # Get the complete list of commits N-version to update
+          COMMIT=$(grep "\-n=" "${COMMIT_ENV_PATH}" | cut -d "=" -f 1)
+
+          for val in ${COMMIT}; do
+            echo "${val}"
+            sed -i "s|${val}=.*|${val}=${{ steps.hash-n.outputs.HASH_N }}|" "${COMMIT_ENV_PATH}"
           done
+
           if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
-            git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+            git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
+            git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
+            git add "${COMMIT_ENV_PATH}" && \
+            git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
+            git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
           else
-            echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
+            echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}"
           fi
 
   update-n-1-version:
@@ -147,61 +161,68 @@ jobs:
         with:
           ref: ${{ env.DIGEST_UPDATER_BRANCH }}
 
-      - name: Fetch digest, and update the params.env file
+      - name: Update the params.env file
         shell: bash
         run: |
           echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}}
-          IMAGES=("odh-minimal-notebook-image-n-1"
-          "odh-minimal-gpu-notebook-image-n-1"
-          "odh-pytorch-gpu-notebook-image-n-1"
-          "odh-generic-data-science-notebook-image-n-1"
-          "odh-tensorflow-gpu-notebook-image-n-1"
-          "odh-trustyai-notebook-image-n-1"
-          "odh-codeserver-notebook-image-n-1"
-          "odh-habana-notebook-image-n-1")
-          REGEXES=("v2-${{ env.RELEASE_VERSION_N_1 }}-\d{8}+-${{ steps.hash-n-1.outputs.HASH_N_1 }}" "cuda-[a-z]+-minimal-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N_1 }}-\d{8}-${{ steps.hash-n-1.outputs.HASH_N_1 }}" "v2-${{ env.RELEASE_VERSION_N_1 }}-\d{8}+-${{ steps.hash-n-1.outputs.HASH_N_1 }}" \
-                   "v2-${{ env.RELEASE_VERSION_N_1 }}-\d{8}+-${{ steps.hash-n-1.outputs.HASH_N_1 }}" "cuda-[a-z]+-tensorflow-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N_1 }}-\d{8}-${{ steps.hash-n-1.outputs.HASH_N_1 }}" "v2-${{ env.RELEASE_VERSION_N_1 }}-\d{8}+-${{ steps.hash-n-1.outputs.HASH_N_1 }}" \
-                   "codeserver-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N_1 }}-\d{8}-${{ steps.hash-n-1.outputs.HASH_N_1 }}"
-                   "habana-[a-z]+-1.10.0-[a-z0-9]+-[a-z]+-3.8-${{ env.RELEASE_VERSION_N_1 }}-\d{8}-${{ steps.hash-n-1.outputs.HASH_N_1 }}")
-
-          for ((i=0;i<${#IMAGES[@]};++i)); do
-            image=${IMAGES[$i]}
-            echo "CHECKING: " $image
+
+          # Get the complete list of images N-1-version to update
+          PARAMS_ENV_PATH="manifests/base/params.env"
+          IMAGES=$(cat "${PARAMS_ENV_PATH}" | grep "\-n-1=" | cut -d "=" -f 1)
+
+          # The order of the regexes array should match with the params.env file
+          REGEXES=("v2-${{ env.RELEASE_VERSION_N_1 }}-\d{8}+-${{ steps.hash-n-1.outputs.HASH_N_1 }}" \
+                  "cuda-[a-z]+-minimal-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N_1 }}-\d{8}-${{ steps.hash-n-1.outputs.HASH_N_1 }}" \
+                  "v2-${{ env.RELEASE_VERSION_N_1 }}-\d{8}+-${{ steps.hash-n-1.outputs.HASH_N_1 }}" \
+                  "v2-${{ env.RELEASE_VERSION_N_1 }}-\d{8}+-${{ steps.hash-n-1.outputs.HASH_N_1 }}" \
+                  "cuda-[a-z]+-tensorflow-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N_1 }}-\d{8}-${{ steps.hash-n-1.outputs.HASH_N_1 }}" \
+                  "v2-${{ env.RELEASE_VERSION_N_1 }}-\d{8}+-${{ steps.hash-n-1.outputs.HASH_N_1 }}" \
+                  "codeserver-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N_1 }}-\d{8}-${{ steps.hash-n-1.outputs.HASH_N_1 }}" \
+                  "habana-[a-z]+-1.10.0-[a-z0-9]+-[a-z]+-3.8-${{ env.RELEASE_VERSION_N_1 }}-\d{8}-${{ steps.hash-n-1.outputs.HASH_N_1 }}")
+
+          i=0
+          for image in ${IMAGES}; do
+            echo "CHECKING: '${image}'"
+            img=$(grep -E "${image}=" "${PARAMS_ENV_PATH}" | cut -d '=' -f2)
+            registry=$(echo "${img}" | cut -d '@' -f1)
             regex=${REGEXES[$i]}
-            img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2)
-            registry=$(echo $img | cut -d '@' -f1)
-            latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
-            digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
-            output=$registry@$digest
-            echo "NEW: " $output
-            sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env
+            skopeo_metadata=$(skopeo inspect --retry-times 3 "docker://${img}")
+            latest_tag=$(echo "${skopeo_metadata}" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
+            digest=$(skopeo inspect --retry-times 3 "docker://${registry}:${latest_tag}" | jq .Digest | tr -d '"')
+            output="${registry}@${digest}"
+            echo "NEW: ${output}"
+            sed -i "s|${image}=.*|${image}=${output}|" "${PARAMS_ENV_PATH}"
+            i=$((i+1))
           done
           if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
-            git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+            git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && \
+            git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && \
+            git add manifests/base/params.env && \
+            git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
+            git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
           else
-            echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
+            echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N_1}}"
           fi
 
-      - name: Fetch digest, and update the commit.env file
+      - name: Update the commit.env file
         run: |
-          echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}}
-          COMMIT=("odh-minimal-notebook-image-commit-n-1"
-           "odh-minimal-gpu-notebook-image-commit-n-1"
-           "odh-pytorch-gpu-notebook-image-commit-n-1"
-           "odh-generic-data-science-notebook-image-commit-n-1"
-           "odh-tensorflow-gpu-notebook-image-commit-n-1"
-           "odh-trustyai-notebook-image-commit-n-1"
-           "odh-codeserver-notebook-image-commit-n-1"
-           "odh-habana-notebook-image-commit-n-1")
-
-          for val in "${COMMIT[@]}"; do
-            echo $val
-            sed -i "s|${val}=.*|${val}=${{ steps.hash-n-1.outputs.HASH_N_1 }}|" manifests/base/commit.env
+          COMMIT_ENV_PATH="manifests/base/commit.env"
+
+          echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1 }}
+          # Get the complete list of images N-1-version to update
+          COMMIT=$(grep "\-n-1=" "${COMMIT_ENV_PATH}" | cut -d "=" -f 1)
+
+          for val in ${COMMIT}; do
+            echo "${val}"
+            sed -i "s|${val}=.*|${val}=${{ steps.hash-n-1.outputs.HASH_N_1 }}|" "${COMMIT_ENV_PATH}"
           done
           if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
-            git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+            git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && \
+            git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && \
+            git add manifests/base/commit.env && \
+            git commit -m "Update image commits for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
           else
-            echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
+            echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N_1}}"
           fi
 
   open-pull-request:

diff --git a/.github/workflows/runtimes-digest-updater.yaml b/.github/workflows/runtimes-digest-updater.yaml
@@ -78,23 +78,29 @@ jobs:
               "jupyter/datascience/ubi9-python-3.9/runtime-images/datascience-ubi9-py39.json"
               "jupyter/datascience/ubi9-python-3.9/runtime-images/pytorch-ubi9-py39.json"
               "jupyter/datascience/ubi9-python-3.9/runtime-images/tensorflow-ubi9-py39.json"
-              "jupyter/datascience/ubi9-python-3.9/runtime-images/ubi9-py39.json")
+              "jupyter/datascience/ubi9-python-3.9/runtime-images/ubi9-py39.json"
+              "jupyter/datascience/ubi9-python-3.9/runtime-images/rocm-tensorflow-ubi9-py39.json"
+              "jupyter/datascience/ubi9-python-3.9/runtime-images/rocm-pytorch-ubi9-py39.json")
 
               for ((i=0;i<${#PATHS[@]};++i)); do
                 path=${PATHS[$i]}
                 img=$(cat ${path} | jq -r '.metadata.image_name')
                 name=$(echo "$path" | sed 's#.*runtime-images/\(.*\)-py.*#\1#')
                 py_version=$(echo "$path" | grep -o 'python-[0-9]\.[0-9]')
                 # Handling specific cases
-                if [[ $name == *tensorflow* ]]; then
+                if [[ $name == tensorflow* ]]; then
                     name="cuda-$name"
                 elif [[ $name == ubi* ]]; then
                     name="minimal-$name"
                 fi
                 registry=$(echo $img | cut -d '@' -f1)
                 regex="runtime-$name-$py_version-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}"
-                echo "CHECKING: "  $regex
                 latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
+                echo "CHECKING: "  $latest_tag
+                if [[ -z "$latest_tag" ]]; then
+                  echo "No matching tag found"
+                  exit 1
+                fi
                 digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
                 output=$registry@$digest
                 echo "NEW: " $output

diff --git a/manifests/base/commit.env b/manifests/base/commit.env
@@ -21,7 +21,7 @@ odh-tensorflow-gpu-notebook-image-commit-n-3=3e71410
 odh-trustyai-notebook-image-commit-n=bcbb42a
 odh-trustyai-notebook-image-commit-n-1=92a1241
 odh-trustyai-notebook-image-commit-n-2=07015ec
-odh-habana-notebook-image-commit-n=bcbb42a
-odh-habana-notebook-image-commit-n-1=92a1241
 odh-codeserver-notebook-image-commit-n=bcbb42a
 odh-codeserver-notebook-image-commit-n-1=92a1241
+odh-habana-notebook-image-commit-n=bcbb42a
+odh-habana-notebook-image-commit-n-1=92a1241
diff --git a/manifests/base/params.env b/manifests/base/params.env
@@ -21,7 +21,7 @@ odh-tensorflow-gpu-notebook-image-n-3=quay.io/modh/cuda-notebooks@sha256:6fadedc
 odh-trustyai-notebook-image-n=quay.io/modh/odh-trustyai-notebook@sha256:73566763063bcbb96c3f51c6c2390df625efed3a295877450064f4ea43723950
 odh-trustyai-notebook-image-n-1=quay.io/modh/odh-trustyai-notebook@sha256:7a5fbf215202b6c2b4a0e022b760490fbd49b71d27ec591faca52ae7c1bbcb41
 odh-trustyai-notebook-image-n-2=quay.io/modh/odh-trustyai-notebook@sha256:8c5e653f6bc6a2050565cf92f397991fbec952dc05cdfea74b65b8fd3047c9d4
-odh-habana-notebook-image-n=quay.io/modh/odh-habana-notebooks@sha256:8974095cf74fc4612eb62b2f72127154f741ec0c35cca8cce9383fff12de6dcc
-odh-habana-notebook-image-n-1=quay.io/modh/odh-habana-notebooks@sha256:118d762d68701260d4b0d3a55b687c292d694da0db1e38ab2f9e8b848f61c9a7
 odh-codeserver-notebook-image-n=quay.io/modh/codeserver@sha256:4dbd8a4fe853337cf35409da59f18b025e6edb52f65c254f2c8e64a311132006
 odh-codeserver-notebook-image-n-1=quay.io/modh/codeserver@sha256:b0619026944c42c99617e1816f04a6d2407c5fae7ddb405332d549cdce69d469
+odh-habana-notebook-image-n=quay.io/modh/odh-habana-notebooks@sha256:8974095cf74fc4612eb62b2f72127154f741ec0c35cca8cce9383fff12de6dcc
+odh-habana-notebook-image-n-1=quay.io/modh/odh-habana-notebooks@sha256:118d762d68701260d4b0d3a55b687c292d694da0db1e38ab2f9e8b848f61c9a7