[main] sync main branch with upstream opendatahub main branch

.github/workflows/params-env.yaml

@@ -1,9 +1,10 @@
 ---
-name: Validation of params.env content (image SHAs)
+name: Validation of image references (image SHAs) in params.env and runtime images
 on:  # yamllint disable-line rule:truthy
   pull_request:
     paths:
       - 'manifests/base/params.env'
+      - 'ci/check-params-env.sh'
 
 permissions:
   contents: read
@@ -21,3 +22,8 @@ jobs:
       - name: Validate the 'manifests/base/params.env' file content
         run: |
           bash ./ci/check-params-env.sh
+
+      - name: Validate references for runtime images
+        id: validate-runtime-images-references
+        run: |
+          bash ./ci/check-runtime-images.sh

Makefile

@@ -415,14 +415,12 @@ test-%: bin/kubectl
 	elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "intel-ml-ubi9"; then \
 		$(call test_with_papermill,intel/ml,ubi9,python-3.9) \
 	elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "trustyai-ubi9"; then \
-		$(MAKE) validate-ubi9-datascience -e FULL_NOTEBOOK_NAME=$(FULL_NOTEBOOK_NAME); \
 		$(call test_with_papermill,trustyai,ubi9,python-3.9) \
 	elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "minimal-ubi8"; then \
 		$(call test_with_papermill,minimal,ubi8,python-3.8) \
 	elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "datascience-ubi8"; then \
 		$(MAKE) validate-ubi8-datascience -e FULL_NOTEBOOK_NAME=$(FULL_NOTEBOOK_NAME); \
 	elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "trustyai-ubi8"; then \
-		$(MAKE) validate-ubi8-datascience -e FULL_NOTEBOOK_NAME=$(FULL_NOTEBOOK_NAME); \
 		$(call test_with_papermill,trustyai,ubi8,python-3.8) \
 	elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "anaconda"; then \
 		echo "There is no test notebook implemented yet for Anaconda Notebook...." \

amd/c9s-python-3.9/Dockerfile

@@ -0,0 +1,102 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+LABEL name="odh-notebook-rocm-python-3.9" \
+      summary="ROCm Python 3.9 base image for ODH notebooks" \
+      description="ROCm Python 3.9 builder image based on CentOS Stream 9 for ODH notebooks" \
+      io.k8s.display-name="ROCm Python 3.9 base image for ODH notebooks" \
+      io.k8s.description="ROCm Python 3.9 builder image based on C9S for ODH notebooks" \
+      authoritative-source-url="https://github.com/opendatahub-io/notebooks" \
+      io.openshift.build.commit.ref="main" \
+      io.openshift.build.source-location="https://github.com/opendatahub-io/notebooks/tree/main/amd/python-3.9" \
+      io.openshift.build.image="quay.io/opendatahub/workbench-images:rocm-python-3.9"
+
+USER 0
+WORKDIR /opt/app-root/bin
+
+ARG ROCM_VERSION=6.1
+ARG AMDGPU_VERSION=6.1
+# Base
+RUN yum -y install git java-1.8.0-openjdk python; yum clean all
+# Install ROCm AMD from:
+# https://github.com/ROCm/ROCm-docker/blob/master/dev/Dockerfile-centos-7-complete
+# Enable epel-release repositories
+RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
+    yum config-manager --set-enabled crb && \
+    yum clean all && rm -rf /var/cache/yum
+
+
+# Install required base build and packaging commands for ROCm
+RUN yum -y install \
+    bc \
+    bridge-utils \
+    glibc.i686 \
+    numactl-libs \
+    libssh \
+    libunwind-devel \
+    libunwind \
+    pciutils \
+    pciutils-libs \
+    re2c \
+    doxygen \
+    elfutils-libelf-devel \
+    expect \
+    numactl-devel \
+    pciutils-devel \
+    qemu-kvm \
+    subversion \
+    dkms \
+    dpkg \
+    dpkg-dev \
+    dpkg-perl && \
+    yum clean all 
+# These binaries are not available.
+# devscripts \
+# pth \
+# libcxx-devel \
+
+
+# Enable the epel repository for fakeroot
+# RUN yum --enablerepo=extras install -y fakeroot
+# RUN yum clean all
+RUN yum install -y fakeroot && \
+    yum clean all
+
+
+# On CentOS, install package centos-release-scl available in CentOS repository:
+# RUN yum install -y centos-release-scl
+
+# Install the devtoolset-7 collection:
+# RUN yum install -y devtoolset-7
+# RUN yum install -y devtoolset-7-libatomic-devel devtoolset-7-elfutils-libelf-devel
+
+# Install the ROCm rpms
+RUN echo "[ROCm]" > /etc/yum.repos.d/rocm.repo && \
+    echo "name=ROCm" >> /etc/yum.repos.d/rocm.repo && \
+    echo "baseurl=https://repo.radeon.com/rocm/el9/$ROCM_VERSION/main" >> /etc/yum.repos.d/rocm.repo && \
+    echo "enabled=1" >> /etc/yum.repos.d/rocm.repo && \
+    echo "gpgcheck=0" >> /etc/yum.repos.d/rocm.repo
+
+RUN echo "[amdgpu]" > /etc/yum.repos.d/amdgpu.repo && \
+    echo "name=amdgpu" >> /etc/yum.repos.d/amdgpu.repo && \
+    echo "baseurl=https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/el/9.2/main/x86_64" >> /etc/yum.repos.d/amdgpu.repo && \
+    echo "enabled=1" >> /etc/yum.repos.d/amdgpu.repo && \
+    echo "gpgcheck=0" >> /etc/yum.repos.d/amdgpu.repo
+
+# Install rocm and amdgpu binaries 
+RUN yum install -y amdgpu-dkms rocm && \
+    yum clean all
+
+# Set ENV to enable devtoolset7 by default 
+# Set ENV to enable devtoolset7 by default 
+# ENV PATH=/opt/rh/devtoolset-7/root/usr/bin:/opt/rocm/hcc/bin:/opt/rocm/hip/bin:/opt/rocm/bin:/opt/rocm/hcc/bin:${PATH:+:${PATH}}
+# ENV MANPATH=/opt/rh/devtoolset-7/root/usr/share/man:${MANPATH}
+# ENV INFOPATH=/opt/rh/devtoolset-7/root/usr/share/info${INFOPATH:+:${INFOPATH}}
+# ENV PCP_DIR=/opt/rh/devtoolset-7/root
+# ENV PERL5LIB=/opt/rh/devtoolset-7/root//usr/lib64/perl5/vendor_perl:/opt/rh/devtoolset-7/root/usr/lib/perl5:/opt/rh/devtoolset-7/root//usr/share/perl5/
+# ENV LD_LIBRARY_PATH=/opt/rocm/lib:/usr/local/lib:/opt/rh/devtoolset-7/root$rpmlibdir$rpmlibdir32${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+# ENV PYTHONPATH=/opt/rh/devtoolset-7/root/usr/lib64/python$pythonvers/site-packages:/opt/rh/devtoolset-7/root/usr/lib/python$pythonvers/
+# ENV LDFLAGS="-Wl,-rpath=/opt/rh/devtoolset-7/root/usr/lib64 -Wl,-rpath=/opt/rh/devtoolset-7/root/usr/lib"
+
+# Restore notebook user workspace
+USER 1001
+WORKDIR /opt/app-root/src

ci/check-params-env.sh

@@ -184,22 +184,21 @@ function check_image_variable_matches_name_and_commitref() {
             ;;
         odh-habana-notebook-image-n)
             expected_name="odh-notebook-habana-jupyter-1.13.0-ubi8-python-3.8"
-            # expected_commitref="2024a"
-            expected_commitref="main"
-            expected_build_name="habana-jupyter-1.13.0-ubi8-python-3.8"
+            expected_commitref="2024a"
+            expected_build_name="habana-jupyter-1.13.0-ubi8-python-3.8-amd64"
             ;;
         odh-habana-notebook-image-n-1)
             expected_name="odh-notebook-habana-jupyter-1.10.0-ubi8-python-3.8"
             # expected_commitref="release-2023b"
             expected_commitref="rhods-1.34"
             expected_build_name="habana-jupyter-1.10.0-ubi8-python-3.8-amd64"
             ;;
-        odh-codeserver-notebook-n)
+        odh-codeserver-notebook-image-n)
             expected_name="odh-notebook-code-server-ubi9-python-3.9"
             expected_commitref="release-2024a"
             expected_build_name="codeserver-ubi9-python-3.9-amd64"
             ;;
-        odh-codeserver-notebook-n-1)
+        odh-codeserver-notebook-image-n-1)
             expected_name="odh-notebook-code-server-ubi9-python-3.9"
             expected_commitref="release-2023b"
             expected_build_name="codeserver-ubi9-python-3.9-amd64"

ci/check-runtime-images.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+#
+# This script serves to check and validate the definitions for runtime images.
+# It does just a brief check of the metadata defined in the json file:
+#   1. checks that given `.metadata.image_name` is valid and can be accessed by skopeo tool
+#   2. checks that tag in `.metadata.tags[0]` can be found in the output from skopeo tool
+#
+# THIS FILE DOESN'T CHECK THAT THE USED LINK TO IMAGE IS THE LATEST ONE AVAILABLE!
+#
+# This script uses `skopeo` and `jq` tools installed locally for retrieving
+# information about the particular remote images.
+#
+# Local execution: ./ci/check-runtime-image.sh
+#   Note: please execute from the root directory so that relative path matches
+#
+# In case of the PR on GitHub, this check is tied to GitHub actions automatically,
+# see `.github/workflows` directory.
+
+# ---------------------------- DEFINED FUNCTIONS ----------------------------- #
+
+function check_image() {
+    local runtime_image_file="${1}"
+
+    echo "---------------------------------------------"
+    echo "Checking file: '${runtime_image_file}'"
+
+    local img_tag
+    local img_url
+    local img_metadata
+
+    img_tag=$(jq -r '.metadata.tags[0]' "${runtime_image_file}") || {
+        echo "ERROR: Couldn't parse image tags metadata for '${runtime_image_file}' runtime image file!"
+        return 1
+    }
+    img_url=$(jq -r '.metadata.image_name' "${runtime_image_file}") || {
+        echo "ERROR: Couldn't parse image URL metadata for '${runtime_image_file}' runtime image file!"
+        return 1
+    }
+
+    img_metadata="$(skopeo inspect --config "docker://${img_url}")" || {
+        echo "ERROR: Couldn't download '${img_url}' image metadata with skopeo tool!"
+        return 1
+    }
+
+    local expected_string="runtime-${img_tag}-ubi"
+    echo "Checking that '${expected_string}' is present in the image metadata"
+    echo "${img_metadata}" | grep --quiet "${expected_string}" || {
+        echo "ERROR: The string '${expected_string}' isn't present in the image metadata at all. Please check that the referenced image '${img_url}' is the correct one!"
+        return 1
+    }
+
+    # TODO: we shall extend this check to check also Label "io.openshift.build.commit.ref" value (e.g. '2024a') or something similar
+}
+
+function main() {
+    ret_code=0
+
+    # If name of the directory isn't good enough, maybe we can improve this to search for the: `"schema_name": "runtime-image"` string.
+    runtime_image_files=$(find . -name "*.json" | grep "runtime-images" | sort --unique)
+
+    IFS=$'\n'
+    for file in ${runtime_image_files}; do
+        check_image "${file}" || {
+            echo "ERROR: Check for '${file}' failed!"
+            ret_code=1
+        }
+    done
+
+    echo "---------------------------------------------"
+    echo ""
+    if test "${ret_code}" -eq 0; then
+        echo "Validation of runtime images definitions was successful! Congrats :)"
+    else
+        echo "ERROR: Some of the runtime image definitions aren't valid, please check above!"
+    fi
+
+    return "${ret_code}"
+}
+
+# ------------------------------ MAIN SCRIPT --------------------------------- #
+
+main
+
+exit "${?}"

ci/security-scan/quay_security_analysis.py

@@ -20,9 +20,9 @@
     "odh-tensorflow-gpu-notebook-image-main",
     "odh-trustyai-notebook-image-main",
     "odh-habana-notebook-image-main",
-    "odh-codeserver-notebook-main",
-    "odh-rstudio-notebook-main",
-    "odh-rstudio-gpu-notebook-main"
+    "odh-codeserver-notebook-image-main",
+    "odh-rstudio-notebook-image-main",
+    "odh-rstudio-gpu-notebook-image-main"
 ]
 
 IMAGES = [
@@ -37,9 +37,9 @@
     "odh-runtime-tensorflow-notebook-image-n",
     "odh-trustyai-notebook-image-n",
     "odh-habana-notebook-image-n",
-    "odh-codeserver-notebook-n",
-    "odh-rstudio-notebook-n",
-    "odh-rstudio-gpu-notebook-n"
+    "odh-codeserver-notebook-image-n",
+    "odh-rstudio-notebook-image-n",
+    "odh-rstudio-gpu-notebook-image-n"
 ]
 
 IMAGES_N_1 = [
@@ -53,9 +53,9 @@
     "odh-tensorflow-gpu-notebook-image-n-1",
     "odh-runtime-tensorflow-notebook-image-n-1",
     "odh-trustyai-notebook-image-n-1",
-    "odh-codeserver-notebook-n-1",
-    "odh-rstudio-notebook-n-1",
-    "odh-rstudio-gpu-notebook-n-1"
+    "odh-codeserver-notebook-image-n-1",
+    "odh-rstudio-notebook-image-n-1",
+    "odh-rstudio-gpu-notebook-image-n-1"
 ]
 
 def generate_markdown_table(branch_dictionary):

ci/security-scan/weekly_commit_ids.env

@@ -18,15 +18,15 @@ odh-trustyai-notebook-image-n=quay.io/opendatahub/workbench-images@sha256:bf2087
 odh-trustyai-notebook-image-n-1=quay.io/opendatahub/workbench-images@sha256:5b5bae7a11f2e34b67726a86d24b8f2c35c701a48d80abbdbc91030033d2fc1f
 odh-habana-notebook-image-main=quay.io/opendatahub/workbench-images@sha256:f5237ad45f84a9adfc5e30d6fab809dcd7fd10dc9048b3c82f8dfe71d2d7eb2c
 odh-habana-notebook-image-n=quay.io/opendatahub/workbench-images@sha256:b0821ae2abe45387a371108ac08e7474b64255e5c4519de5da594b4617fd79fe
-odh-codeserver-notebook-main=quay.io/opendatahub/workbench-images@sha256:2797380eaf0f05d6002e9fbb41a6a8b5368b658230ba46b07341c9c96797f591
-odh-codeserver-notebook-n=quay.io/opendatahub/workbench-images@sha256:1c5bcbfc222dfb59849fee67e050719c688c93d3608f7b46edbe5666263641f3
-odh-codeserver-notebook-n-1=quay.io/opendatahub/workbench-images@sha256:fd5b9f65c0f46d4c093e2f58fce305eeb125bf19ee1d88f67b9fafe56142e92d
-odh-rstudio-notebook-main=quay.io/opendatahub/workbench-images@sha256:cffcf81ca0dba140d3dfc5ab452eebd6db92e55da5bdfbe3f931661489a8a596
-odh-rstudio-notebook-n=quay.io/opendatahub/workbench-images@sha256:8e99e4e3800db121d02b50adec5eba27746bf89d32dba3e2b17e8d750ac53608
-odh-rstudio-notebook-n-1=quay.io/opendatahub/workbench-images@sha256:75d6764e1155c1d18dc4472ff319f9291d0d9703b19ee1374e902b6ab7f55cfb
-odh-rstudio-gpu-notebook-main=quay.io/opendatahub/workbench-images@sha256:41d07177990519db629796f743b6dcb663bc8090e4c8248348f746b2fa4f7dbb
-odh-rstudio-gpu-notebook-n=quay.io/opendatahub/workbench-images@sha256:3ad0bb5f3b8c2ca1d29a423913b6d8f32353d9787c5f38e4b56a9a922f6e3cdd
-odh-rstudio-gpu-notebook-n-1=quay.io/opendatahub/workbench-images@sha256:aef5fd12264651abf286e9a4efbe25ca002cc257fbc6f1a5daf39fd55c7d6206
+odh-codeserver-notebook-image-main=quay.io/opendatahub/workbench-images@sha256:2797380eaf0f05d6002e9fbb41a6a8b5368b658230ba46b07341c9c96797f591
+odh-codeserver-notebook-image-n=quay.io/opendatahub/workbench-images@sha256:1c5bcbfc222dfb59849fee67e050719c688c93d3608f7b46edbe5666263641f3
+odh-codeserver-notebook-image-n-1=quay.io/opendatahub/workbench-images@sha256:fd5b9f65c0f46d4c093e2f58fce305eeb125bf19ee1d88f67b9fafe56142e92d
+odh-rstudio-notebook-image-main=quay.io/opendatahub/workbench-images@sha256:cffcf81ca0dba140d3dfc5ab452eebd6db92e55da5bdfbe3f931661489a8a596
+odh-rstudio-notebook-image-n=quay.io/opendatahub/workbench-images@sha256:8e99e4e3800db121d02b50adec5eba27746bf89d32dba3e2b17e8d750ac53608
+odh-rstudio-notebook-image-n-1=quay.io/opendatahub/workbench-images@sha256:75d6764e1155c1d18dc4472ff319f9291d0d9703b19ee1374e902b6ab7f55cfb
+odh-rstudio-gpu-notebook-image-main=quay.io/opendatahub/workbench-images@sha256:41d07177990519db629796f743b6dcb663bc8090e4c8248348f746b2fa4f7dbb
+odh-rstudio-gpu-notebook-image-n=quay.io/opendatahub/workbench-images@sha256:3ad0bb5f3b8c2ca1d29a423913b6d8f32353d9787c5f38e4b56a9a922f6e3cdd
+odh-rstudio-gpu-notebook-image-n-1=quay.io/opendatahub/workbench-images@sha256:aef5fd12264651abf286e9a4efbe25ca002cc257fbc6f1a5daf39fd55c7d6206
 odh-runtime-minimal-notebook-image-main=quay.io/opendatahub/workbench-images@sha256:b02d8970449a48362a9f54ea563692b8d4c0e9f1f689ea1cf6bd2da18538a421
 odh-runtime-minimal-notebook-image-n=quay.io/opendatahub/workbench-images@sha256:91224cde193645c231e454bdcc25ab1aa40dd7c7bc466c87baffa8c03f5e3128
 odh-runtime-minimal-notebook-image-n-1=quay.io/opendatahub/workbench-images@sha256:41dd881199fd93ffccc4f00c16a69ad16f27f1e4877373ad96ff7a94b9564972

codeserver/ubi9-python-3.9/Dockerfile

@@ -39,6 +39,13 @@ RUN yum install -y "https://github.com/coder/code-server/releases/download/${COD
 RUN chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \
     fix-permissions /opt/app-root -P
 
+COPY --chown=1001:0 utils utils/
+
+# Create and intall the extensions though build-time on a temporary directory. Later this directory will copied on the `/opt/app-root/src/.local/share/code-server/extensions` via run-code-server.sh file when it starts up. 
+RUN mkdir -p /opt/app-root/extensions-temp && \
+    code-server --install-extension /opt/app-root/bin/utils/ms-python.python-2024.2.1.vsix --extensions-dir /opt/app-root/extensions-temp && \
+    code-server --install-extension  /opt/app-root/bin/utils/ms-toolsai.jupyter-2023.9.100.vsix --extensions-dir /opt/app-root/extensions-temp
+
 # Install NGINX to proxy code-server and pass probes check
 ENV NGINX_VERSION=1.24 \
     NGINX_SHORT_VER=124 \
@@ -88,15 +95,20 @@ RUN sed -i -f ${NGINX_APP_ROOT}/nginxconf.sed ${NGINX_CONF_PATH} && \
     chmod -R ug+rwX ${NGINX_APP_ROOT}/etc && \
     chmod -R ug+rwX ${NGINX_CONTAINER_SCRIPTS_PATH}/nginx-start && \
     chmod -R ug+rwX /var/lib/nginx /var/log/nginx /run && \
-    rpm-file-permissions
+    rpm-file-permissions && \
+    # Ensure the temporary directory and target directory have the correct permissions
+    mkdir -p /opt/app-root/src/.local/share/code-server/extensions && \
+    mkdir -p /opt/app-root/src/.local/share/code-server/coder-logs && \
+    chown -R 1001:0 /opt/app-root/src/.local/share/code-server && \
+    chown -R 1001:0 /opt/app-root/extensions-temp && \
+    chown -R 1001:0 /opt/app-root/src/.config/code-server
 
 ## Configure nginx
 COPY nginx/serverconf/ /opt/app-root/etc/nginx.default.d/
 COPY nginx/httpconf/ /opt/app-root/etc/nginx.d/
 COPY nginx/api/ /opt/app-root/api/
 
 # Launcher
-COPY --chown=1001:0 utils utils/
 COPY --chown=1001:0 run-code-server.sh run-nginx.sh ./
 
 ENV SHELL /bin/bash