Merge pull request #340 from harshad16/rsync-main

[main] sync main from opendatahub-io:main

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -22,6 +22,8 @@ jobs:
     env:
       # GitHub image registry used for storing $(CONTAINER_ENGINE)'s cache
       CACHE: "ghcr.io/${{ github.repository }}/workbench-images/build-cache"
+      # Targets (and their folder) that should be scanned using FS instead of IMAGE scan due to resource constraints
+      TRIVY_SCAN_FS_JSON: '{"rocm-jupyter-pytorch-ubi9-python-3.9": "jupyter/rocm/pytorch/ubi9-python-3.9"}'
 
     steps:
 
@@ -113,10 +115,12 @@ jobs:
 
       # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push
       - name: "push|schedule: make ${{ inputs.target }}"
-        run: "make ${{ inputs.target }}"
+        run: |
+          SANITIZED_REF_NAME=$(echo "${{ github.ref_name }}" | sed 's/[^a-zA-Z0-9._-]/_/g')
+          export IMAGE_TAG="${SANITIZED_REF_NAME}_${{ github.sha }}"
+          make ${{ inputs.target }}
         if: ${{ fromJson(inputs.github).event_name == 'push' || fromJson(inputs.github).event_name == 'schedule' }}
         env:
-          IMAGE_TAG: "${{ github.ref_name }}_${{ github.sha }}"
           IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images"
           CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }} --cache-to ${{ env.CACHE }}"
 
@@ -138,27 +142,42 @@ jobs:
       - name: "Show podman images information"
         run: podman images
 
-      - name: "pull_request|schedule: resolve image name if Trivy scan should run"
-        id: resolve-image
+      - name: "pull_request|schedule: resolve target if Trivy scan should run"
+        id: resolve-target
         if: ${{ fromJson(inputs.github).event_name == 'pull_request' || fromJson(inputs.github).event_name == 'schedule' }}
         env:
           EVENT_NAME: ${{ fromJson(inputs.github).event_name }}
           HAS_TRIVY_LABEL: ${{ contains(fromJson(inputs.github).event.pull_request.labels.*.name, 'trivy-scan') }}
+          FS_SCAN_FOLDER: ${{ fromJson(env.TRIVY_SCAN_FS_JSON)[inputs.target] }}
         run: |
           if [[ "$EVENT_NAME" == "pull_request" && "$HAS_TRIVY_LABEL" == "true" ]]; then
-            IMAGE_NAME="localhost:5000/workbench-images:${{ inputs.target }}-${{ github.sha }}"
-            echo "image=$IMAGE_NAME" >> $GITHUB_OUTPUT
+            if [[ -n "$FS_SCAN_FOLDER" ]]; then
+              TARGET="$FS_SCAN_FOLDER"
+              TYPE="fs"
+            else
+              TARGET="localhost:5000/workbench-images:${{ inputs.target }}-${{ github.sha }}"
+              TYPE="image"
+            fi
           elif [[ "$EVENT_NAME" == "schedule" ]]; then
-            IMAGE_NAME="ghcr.io/${{ github.repository }}/workbench-images:${{ inputs.target }}-${{ github.ref_name }}_${{ github.sha }}"
-            echo "image=$IMAGE_NAME" >> $GITHUB_OUTPUT
+            if [[ -n "$FS_SCAN_FOLDER" ]]; then
+              TARGET="$FS_SCAN_FOLDER"
+              TYPE="fs"
+            else
+              TARGET="ghcr.io/${{ github.repository }}/workbench-images:${{ inputs.target }}-${{ github.ref_name }}_${{ github.sha }}"
+              TYPE="image"
+            fi
           fi
 
-          if [[ -z "$IMAGE_NAME" ]]; then
+          if [[ -n "$TARGET" ]]; then
+            echo "target=$TARGET" >> $GITHUB_OUTPUT
+            echo "type=$TYPE" >> $GITHUB_OUTPUT
+            echo "Trivy scan will run on $TARGET ($TYPE)"
+          else
             echo "Trivy scan won't run"
           fi
 
       - name: Run Trivy vulnerability scanner
-        if: ${{ steps.resolve-image.outputs.image }}
+        if: ${{ steps.resolve-target.outputs.target }}
         run: |
           TRIVY_VERSION=0.53.0
           REPORT_FOLDER=${{ github.workspace }}/report
@@ -168,22 +187,31 @@ jobs:
           mkdir -p $REPORT_FOLDER
           cp ci/$REPORT_TEMPLATE $REPORT_FOLDER
 
-          IMAGE_NAME=${{ steps.resolve-image.outputs.image }}
-          echo "Scanning $IMAGE_NAME"
+          SCAN_TARGET=${{ steps.resolve-target.outputs.target }}
+          SCAN_TYPE=${{ steps.resolve-target.outputs.type }}
+          echo "Scanning $SCAN_TARGET ($SCAN_TYPE)"
+
+          if [[ "$SCAN_TYPE" == "image" ]]; then
+            SCAN_ARGS="--image-src podman --podman-host /var/run/podman/podman.sock"
+            PODMAN_ARGS="-v ${PODMAN_SOCK}:/var/run/podman/podman.sock"
+          elif [[ "$SCAN_TYPE" == "fs" ]]; then
+            WORKSPACE_FOLDER="/workspace"
+            SCAN_TARGET="$WORKSPACE_FOLDER/$SCAN_TARGET"
+            PODMAN_ARGS="-v ${{ github.workspace }}:$WORKSPACE_FOLDER"
+          fi
 
           # have trivy access podman socket,
           # https://github.com/aquasecurity/trivy/issues/580#issuecomment-666423279
           podman run --rm \
-              -v ${PODMAN_SOCK}:/var/run/podman/podman.sock \
+              $PODMAN_ARGS \
               -v ${REPORT_FOLDER}:/report \
               docker.io/aquasec/trivy:$TRIVY_VERSION \
-                image \
-                --image-src podman \
-                --podman-host /var/run/podman/podman.sock \
+                $SCAN_TYPE \
+                $SCAN_ARGS \
                 --scanners vuln --ignore-unfixed \
                 --exit-code 0 --timeout 30m \
                 --format template --template "@/report/$REPORT_TEMPLATE" -o /report/$REPORT_FILE \
-                $IMAGE_NAME
+                $SCAN_TARGET
 
           cat $REPORT_FOLDER/$REPORT_FILE >> $GITHUB_STEP_SUMMARY
 

.github/workflows/build-notebooks.yaml

@@ -471,6 +471,28 @@
             },
             "secrets": "inherit"
         },
+        "rocm-runtime-pytorch-ubi9-python-3_9": {
+            "needs": [
+                "rocm-ubi9-python-3_9"
+            ],
+            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
+            "with": {
+                "target": "rocm-runtime-pytorch-ubi9-python-3.9",
+                "github": "${{ toJSON(github) }}"
+            },
+            "secrets": "inherit"
+        },
+        "rocm-runtime-tensorflow-ubi9-python-3_9": {
+            "needs": [
+                "rocm-ubi9-python-3_9"
+            ],
+            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
+            "with": {
+                "target": "rocm-runtime-tensorflow-ubi9-python-3.9",
+                "github": "${{ toJSON(github) }}"
+            },
+            "secrets": "inherit"
+        },
         "base-anaconda-python-3_8": {
             "needs": [],
             "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",

Makefile

@@ -284,6 +284,16 @@ rocm-jupyter-tensorflow-ubi9-python-3.9: rocm-jupyter-datascience-ubi9-python-3.
 rocm-jupyter-pytorch-ubi9-python-3.9: rocm-jupyter-datascience-ubi9-python-3.9
 	$(call image,$@,jupyter/rocm/pytorch/ubi9-python-3.9,$<)
 
+# Build and push rocm-jupyter-runtime-pytorch-ubi9-python-3.9 image to the registry
+.PHONY: rocm-runtime-pytorch-ubi9-python-3.9
+rocm-runtime-pytorch-ubi9-python-3.9: rocm-ubi9-python-3.9
+	$(call image,$@,runtimes/rocm-pytorch/ubi9-python-3.9,$<)
+
+# Build and push rocm-jupyter-runtime-tensorflow-ubi9-python-3.9 image to the registry
+.PHONY: rocm-runtime-tensorflow-ubi9-python-3.9
+rocm-runtime-tensorflow-ubi9-python-3.9: rocm-ubi9-python-3.9
+	$(call image,$@,runtimes/rocm-tensorflow/ubi9-python-3.9,$<)
+
 ####################################### Buildchain for Anaconda Python #######################################
 
 # Build and push base-anaconda-python-3.8 image to the registry
@@ -553,13 +563,30 @@ validate-rstudio-image: bin/kubectl
 # This is only for the workflow action
 .PHONY: refresh-pipfilelock-files
 refresh-pipfilelock-files:
-	cd jupyter/rocm/tensorflow/ubi9-python-3.9 && rm -rf Pipfile.lock && pipenv lock
-	cd runtimes/datascience/ubi8-python-3.8 && rm -rf Pipfile.lock && pipenv lock
-	cd runtimes/datascience/ubi9-python-3.9 && rm -rf Pipfile.lock && pipenv lock
-	cd runtimes/pytorch/ubi9-python-3.9 && rm -rf Pipfile.lock && pipenv lock
-	cd runtimes/pytorch/ubi8-python-3.8 && rm -rf Pipfile.lock && pipenv lock
-	cd runtimes/tensorflow/ubi8-python-3.8 && rm -rf Pipfile.lock && pipenv lock
-	cd runtimes/tensorflow/ubi9-python-3.9 && rm -rf Pipfile.lock && pipenv lock
+	cd base/ubi8-python-3.8 && pipenv lock
+	cd base/ubi9-python-3.9 && pipenv lock
+	cd base/c9s-python-3.9 && pipenv lock
+	cd jupyter/minimal/ubi8-python-3.8 && pipenv lock
+	cd jupyter/minimal/ubi9-python-3.9 && pipenv lock
+	cd jupyter/datascience/ubi8-python-3.8 && pipenv lock
+	cd jupyter/datascience/ubi9-python-3.9 && pipenv lock
+	cd jupyter/pytorch/ubi9-python-3.9 && pipenv lock
+	cd jupyter/tensorflow/ubi9-python-3.9 && pipenv lock
+	cd jupyter/trustyai/ubi9-python-3.9 && pipenv lock
+	cd habana/1.10.0/ubi8-python-3.8 && pipenv lock
+	cd habana/1.13.0/ubi8-python-3.8 && pipenv lock
+	cd runtimes/minimal/ubi8-python-3.8 && pipenv lock
+	cd runtimes/minimal/ubi9-python-3.9 && pipenv lock
+	cd runtimes/datascience/ubi8-python-3.8 && pipenv lock
+	cd runtimes/datascience/ubi9-python-3.9 && pipenv lock
+	cd runtimes/pytorch/ubi9-python-3.9 && pipenv lock
+	cd runtimes/pytorch/ubi8-python-3.8 && pipenv lock
+	cd runtimes/tensorflow/ubi8-python-3.8 && pipenv lock
+	cd runtimes/tensorflow/ubi9-python-3.9 && pipenv lock
+	cd runtimes/rocm-tensorflow/ubi9-python-3.9 && pipenv lock
+	cd runtimes/rocm-pytorch/ubi9-python-3.9 && pipenv lock
+
+
 
 # This is only for the workflow action
 # For running manually, set the required environment variables

habana/1.10.0/ubi8-python-3.8/Pipfile

@@ -33,7 +33,7 @@ pybind11 = "==2.6.2"
 elyra-pipeline-editor-extension = "~=3.15.0"
 elyra-python-editor-extension = "~=3.15.0"
 elyra-code-snippet-extension = "~=3.15.0"
-kfp-tekton = "<1.6.0"
+kfp-tekton = "==1.5.7" # pinned to avoid upgrade of pyyaml which is required.
 boto3 = "~=1.26.165"
 kafka-python = "~=2.0.2"
 matplotlib = "~=3.6.3"
@@ -45,18 +45,24 @@ scipy = "~=1.10.1"
 jupyter-bokeh = "~=3.0.5"
 jupyterlab-lsp = "~=3.10.2"
 jupyterlab-widgets = "~=3.0.5"
-jupyter-resource-usage = "~=0.6.0"
+jupyter-resource-usage = "~=0.7.2"
 
 # Parent image requirements to maintain cohesion
 jupyterlab = "~=3.5.3"
 jupyter-server = "~=2.1.0"
-jupyter-server-proxy = "~=3.2.2"
+jupyter-server-proxy = "~=3.2.4"
 jupyter-server-terminals = "~=0.4.4"
 jupyterlab-git = "~=0.41.0"
 nbdime = "~=3.1.1"
 nbgitpuller = "~=1.1.1"
 # ---
 wheel = "~=0.38.4"
 
+# conflict between python-lsp-server and following packages
+# due to pycodestyle dependency, pinning this, till we get 
+# a fix from python-lsp-server 
+autopep8 = "~=2.0.4"
+flake8 = "~=7.0.0"
+
 [requires]
 python_version = "3.8"