diff --git a/.github/workflows/build-notebooks-TEMPLATE.yaml b/.github/workflows/build-notebooks-TEMPLATE.yaml index f0ac71272..9443f8782 100644 --- a/.github/workflows/build-notebooks-TEMPLATE.yaml +++ b/.github/workflows/build-notebooks-TEMPLATE.yaml @@ -23,7 +23,7 @@ jobs: # GitHub image registry used for storing $(CONTAINER_ENGINE)'s cache CACHE: "ghcr.io/${{ github.repository }}/workbench-images/build-cache" # Targets (and their folder) that should be scanned using FS instead of IMAGE scan due to resource constraints - TRIVY_SCAN_FS_JSON: '{"rocm-jupyter-pytorch-ubi9-python-3.9": "jupyter/rocm/pytorch/ubi9-python-3.9"}' + TRIVY_SCAN_FS_JSON: '{}' steps: @@ -123,24 +123,23 @@ jobs: env: IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images" CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }} --cache-to ${{ env.CACHE }}" + # dependent images were already built and pushed, so just let podman pull it + BUILD_DEPENDENT_IMAGES: "no" # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request - name: "pull_request: make ${{ inputs.target }}" run: | - # start a black hole container registry as make target always does a push - mkdir -p $HOME/.config/containers/registries.conf.d/ - cp ci/cached-builds/insecure_localhost_registry.conf $HOME/.config/containers/registries.conf.d/insecure_localhost_registry.conf - go run ci/cached-builds/dev_null_container_registry.go & - # build and push the image make ${{ inputs.target }} if: "${{ fromJson(inputs.github).event_name == 'pull_request' }}" env: IMAGE_TAG: "${{ github.sha }}" IMAGE_REGISTRY: "localhost:5000/workbench-images" CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }}" + # We don't have access to image registry, so disable pushing + PUSH_IMAGES: "no" - name: "Show podman images information" - run: podman images + run: podman images --digests - name: "pull_request|schedule: resolve target if Trivy scan should run" id: resolve-target diff --git a/Makefile b/Makefile index b33b7e1fa..20ecef69d 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,10 @@ IMAGE_REGISTRY ?= quay.io/opendatahub/workbench-images RELEASE ?= 2024a # additional user-specified caching parameters for $(CONTAINER_ENGINE) build CONTAINER_BUILD_CACHE_ARGS ?= --no-cache +# whether to build all dependent images or just the one specified +BUILD_DEPENDENT_IMAGES ?= yes +# whether to push the images to a registry as they are built +PUSH_IMAGES ?= yes # OS dependant: Generate date, select appropriate cmd to locate container engine ifeq ($(OS), Windows_NT) @@ -58,10 +62,19 @@ endef # ARG 1: Image tag name. # ARG 2: Path of image context we want to build. # ARG 3: Base image tag name (optional). +# +# BUILD_DEPENDENT_IMAGES: only build images that were explicitly given as a goal on command line +# PUSH_IMAGES: allows skipping podman push define image $(info #*# Image build directory: <$(2)> #(MACHINE-PARSED LINE)#*#...) - $(call build_image,$(1),$(2),$(3)) - $(call push_image,$(1)) + + $(if $(or $(BUILD_DEPENDENT_IMAGES:no=), $(filter $@,$(MAKECMDGOALS))), + $(call build_image,$(1),$(2),$(3)) + + $(if $(PUSH_IMAGES:no=), + $(call push_image,$(1)) + ) + ) endef ####################################### Buildchain for Python 3.8 using ubi8 ####################################### diff --git a/README.md b/README.md index 4eb69624a..490d30b24 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,9 @@ To build a workbench image, you can execute the following command: make ${WORKBENCH_NAME} -e IMAGE_REGISTRY=quay.io/${YOUR_USER}/workbench-images -e RELEASE=2023x ``` -Using `IMAGE_REGISTRY` and `RELEASE` variables you can overwrite the default values and use a different registry or release tag +Using `IMAGE_REGISTRY` and `RELEASE` variables you can overwrite the default values and use a different registry or release tag + +Using `CONTAINER_BUILD_CACHE_ARGS` (default: `--no-cache`), `BUILD_DEPENDENT_IMAGES`, and `PUSH_IMAGES` variables you can further customize the build process. ### Local Execution diff --git a/ci/cached-builds/dev_null_container_registry.go b/ci/cached-builds/dev_null_container_registry.go deleted file mode 100644 index bb3049fc0..000000000 --- a/ci/cached-builds/dev_null_container_registry.go +++ /dev/null @@ -1,14 +0,0 @@ -package main - -import ( - "log" - "net/http" -) - -func main() { - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - log.Printf("%s %v", r.Method, r.URL) - }) - - log.Fatal(http.ListenAndServe(":5000", nil)) -} diff --git a/ci/cached-builds/insecure_localhost_registry.conf b/ci/cached-builds/insecure_localhost_registry.conf deleted file mode 100644 index cddc459e8..000000000 --- a/ci/cached-builds/insecure_localhost_registry.conf +++ /dev/null @@ -1,3 +0,0 @@ -[[registry]] -location = "localhost:5000" -insecure = true diff --git a/jupyter/datascience/ubi9-python-3.9/Dockerfile b/jupyter/datascience/ubi9-python-3.9/Dockerfile index 08bf639da..878ff2009 100644 --- a/jupyter/datascience/ubi9-python-3.9/Dockerfile +++ b/jupyter/datascience/ubi9-python-3.9/Dockerfile @@ -20,7 +20,10 @@ COPY setup-elyra.sh ./utils/ RUN echo "Installing softwares and packages" && \ micropipenv install && \ - rm -f ./Pipfile.lock + rm -f ./Pipfile.lock && \ + # Fix permissions to support pip in Openshift environments \ + chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \ + fix-permissions /opt/app-root -P COPY utils ./utils/ diff --git a/jupyter/minimal/ubi8-python-3.8/Dockerfile b/jupyter/minimal/ubi8-python-3.8/Dockerfile index 7b3d8f2e6..c8361c5c0 100644 --- a/jupyter/minimal/ubi8-python-3.8/Dockerfile +++ b/jupyter/minimal/ubi8-python-3.8/Dockerfile @@ -18,14 +18,12 @@ COPY utils utils/ COPY Pipfile.lock start-notebook.sh ./ # Install Python dependencies from Pipfile.lock file -RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock - -# Disable announcement plugin of jupyterlab -RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements" - -# Fix permissions to support pip in Openshift environments -RUN chmod -R g+w /opt/app-root/lib/python3.8/site-packages && \ - fix-permissions /opt/app-root -P +RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock && \ + # Disable announcement plugin of jupyterlab \ + jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \ + # Fix permissions to support pip in Openshift environments \ + chmod -R g+w /opt/app-root/lib/python3.8/site-packages && \ + fix-permissions /opt/app-root -P WORKDIR /opt/app-root/src diff --git a/jupyter/minimal/ubi9-python-3.9/Dockerfile b/jupyter/minimal/ubi9-python-3.9/Dockerfile index 37befa970..573a59b04 100644 --- a/jupyter/minimal/ubi9-python-3.9/Dockerfile +++ b/jupyter/minimal/ubi9-python-3.9/Dockerfile @@ -18,14 +18,12 @@ COPY utils utils/ COPY Pipfile.lock start-notebook.sh ./ # Install Python dependencies from Pipfile.lock file -RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock - -# Disable announcement plugin of jupyterlab -RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements" - -# Fix permissions to support pip in Openshift environments -RUN chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \ - fix-permissions /opt/app-root -P +RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock && \ + # Disable announcement plugin of jupyterlab \ + jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \ + # Fix permissions to support pip in Openshift environments \ + chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \ + fix-permissions /opt/app-root -P WORKDIR /opt/app-root/src diff --git a/jupyter/pytorch/ubi9-python-3.9/Dockerfile b/jupyter/pytorch/ubi9-python-3.9/Dockerfile index dc5557385..e8baaa475 100644 --- a/jupyter/pytorch/ubi9-python-3.9/Dockerfile +++ b/jupyter/pytorch/ubi9-python-3.9/Dockerfile @@ -20,8 +20,7 @@ RUN echo "Installing softwares and packages" && \ # Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y \ sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json && \ # Disable announcement plugin of jupyterlab \ - jupyter labextension disable "@jupyterlab/apputils-extension:announcements" - -# Fix permissions to support pip in Openshift environments -RUN chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \ + jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \ + # Fix permissions to support pip in Openshift environments \ + chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \ fix-permissions /opt/app-root -P diff --git a/jupyter/rocm/pytorch/ubi9-python-3.9/Dockerfile b/jupyter/rocm/pytorch/ubi9-python-3.9/Dockerfile index ca883c0fb..fc3c1ee3a 100644 --- a/jupyter/rocm/pytorch/ubi9-python-3.9/Dockerfile +++ b/jupyter/rocm/pytorch/ubi9-python-3.9/Dockerfile @@ -13,11 +13,16 @@ LABEL name="odh-notebook-jupyter-rocm-pytorch-ubi9-python-3.9" \ # Install Python packages and Jupyterlab extensions from Pipfile.lock COPY Pipfile.lock ./ +# Copy utility script +COPY de-vendor-torch.sh ./ RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock && \ + # De-vendor the ROCm libs that are embedded in Pytorch \ + ./de-vendor-torch.sh && \ + rm ./de-vendor-torch.sh && \ # Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json && \ - # Disable announcement plugin of jupyterlab + # Disable announcement plugin of jupyterlab \ jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \ - # Fix permissions to support pip in Openshift environments + # Fix permissions to support pip in Openshift environments \ chmod -R g+w /opt/app-root/lib/python3.9/site-packages && fix-permissions /opt/app-root -P diff --git a/jupyter/rocm/pytorch/ubi9-python-3.9/de-vendor-torch.sh b/jupyter/rocm/pytorch/ubi9-python-3.9/de-vendor-torch.sh new file mode 100755 index 000000000..780f8815e --- /dev/null +++ b/jupyter/rocm/pytorch/ubi9-python-3.9/de-vendor-torch.sh @@ -0,0 +1,41 @@ +#!/bin/sh +set -ex +# Replace PyTorch's vendored shared libraries with system libraries +# The script assumes that PyTorch is built with the same ROCm ABI as the +# system installation of ROCm. + +# Source: https://github.com/tiran/instructlab-containers/blob/main/containers/rocm/de-vendor-torch.sh + +PYTHON=python3.9 +ROCMLIB=/opt/rocm/lib +TORCHLIB=/opt/app-root/lib/${PYTHON}/site-packages/torch/lib + +ln -sf /usr/lib64/libdrm.so.2 ${TORCHLIB}/libdrm.so +ln -sf /usr/lib64/libdrm_amdgpu.so.1 ${TORCHLIB}/libdrm_amdgpu.so + +ln -sf ${ROCMLIB}/libamd_comgr.so.2 ${TORCHLIB}/libamd_comgr.so +ln -sf ${ROCMLIB}/libamdhip64.so.6 ${TORCHLIB}/libamdhip64.so +ln -sf ${ROCMLIB}/libhipblaslt.so.0 ${TORCHLIB}/libhipblaslt.so +ln -sf ${ROCMLIB}/libhipblas.so.2 ${TORCHLIB}/libhipblas.so +ln -sf ${ROCMLIB}/libhipfft.so.0 ${TORCHLIB}/libhipfft.so +ln -sf ${ROCMLIB}/libhiprand.so.1 ${TORCHLIB}/libhiprand.so +ln -sf ${ROCMLIB}/libhiprtc.so.6 ${TORCHLIB}/libhiprtc.so +ln -sf ${ROCMLIB}/libhipsolver.so.0 ${TORCHLIB}/libhipsolver.so +ln -sf ${ROCMLIB}/libhipsparse.so.1 ${TORCHLIB}/libhipsparse.so +ln -sf ${ROCMLIB}/libhsa-runtime64.so.1 ${TORCHLIB}/libhsa-runtime64.so +ln -sf ${ROCMLIB}/libMIOpen.so.1 ${TORCHLIB}/libMIOpen.so +ln -sf ${ROCMLIB}/librccl.so.1 ${TORCHLIB}/librccl.so +ln -sf ${ROCMLIB}/librocblas.so.4 ${TORCHLIB}/librocblas.so +ln -sf ${ROCMLIB}/librocfft.so.0 ${TORCHLIB}/librocfft.so +ln -sf ${ROCMLIB}/librocm_smi64.so.6 ${TORCHLIB}/librocm_smi64.so +ln -sf ${ROCMLIB}/librocrand.so.1 ${TORCHLIB}/librocrand.so +ln -sf ${ROCMLIB}/librocsolver.so.0 ${TORCHLIB}/librocsolver.so +ln -sf ${ROCMLIB}/librocsparse.so.1 ${TORCHLIB}/librocsparse.so +ln -sf ${ROCMLIB}/libroctracer64.so.4 ${TORCHLIB}/libroctracer64.so +ln -sf ${ROCMLIB}/libroctx64.so.4 ${TORCHLIB}/libroctx64.so + +rm -rf ${TORCHLIB}/rocblas +ln -sf ${ROCMLIB}/rocblas ${TORCHLIB}/rocblas + +rm -rf ${TORCHLIB}/hipblaslt +ln -sf ${ROCMLIB}/hipblaslt ${TORCHLIB}/hipblaslt diff --git a/jupyter/rocm/tensorflow/ubi9-python-3.9/Dockerfile b/jupyter/rocm/tensorflow/ubi9-python-3.9/Dockerfile index 87acf24d2..721e6bd6d 100644 --- a/jupyter/rocm/tensorflow/ubi9-python-3.9/Dockerfile +++ b/jupyter/rocm/tensorflow/ubi9-python-3.9/Dockerfile @@ -15,9 +15,9 @@ LABEL name="odh-notebook-jupyter-rocm-tensorflow-ubi9-python-3.9" \ COPY Pipfile.lock ./ RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock && \ - # Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y + # Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y \ sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json && \ - # Disable announcement plugin of jupyterlab + # Disable announcement plugin of jupyterlab \ jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \ - # Fix permissions to support pip in Openshift environments + # Fix permissions to support pip in Openshift environments \ chmod -R g+w /opt/app-root/lib/python3.9/site-packages && fix-permissions /opt/app-root -P diff --git a/jupyter/tensorflow/ubi9-python-3.9/Dockerfile b/jupyter/tensorflow/ubi9-python-3.9/Dockerfile index 31579fdb8..82712e34c 100644 --- a/jupyter/tensorflow/ubi9-python-3.9/Dockerfile +++ b/jupyter/tensorflow/ubi9-python-3.9/Dockerfile @@ -19,14 +19,11 @@ RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./P # Temporary:Workaround for fixing the kfp-kubernetes 1.0.0 for elyra pipeline execution # TODO: Remove this patch once the issue is fixed with kfp-kubernetes upgrade. -RUN patch /opt/app-root/lib/python3.9/site-packages/elyra/templates/kubeflow/v2/python_dsl_template.jinja2 -i utils/python_dsl_template.patch - -# Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y -RUN sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json - -# Disable announcement plugin of jupyterlab -RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements" - -# Fix permissions to support pip in Openshift environments -RUN chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \ +RUN patch /opt/app-root/lib/python3.9/site-packages/elyra/templates/kubeflow/v2/python_dsl_template.jinja2 -i utils/python_dsl_template.patch && \ + # Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y \ + sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json && \ + # Disable announcement plugin of jupyterlab \ + jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \ + # Fix permissions to support pip in Openshift environments \ + chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \ fix-permissions /opt/app-root -P diff --git a/jupyter/trustyai/ubi9-python-3.9/Dockerfile b/jupyter/trustyai/ubi9-python-3.9/Dockerfile index 3dc7e3512..24f7d4cb5 100644 --- a/jupyter/trustyai/ubi9-python-3.9/Dockerfile +++ b/jupyter/trustyai/ubi9-python-3.9/Dockerfile @@ -23,14 +23,11 @@ USER 1001 # Install Python packages and Jupyterlab extensions from Pipfile.lock COPY Pipfile.lock ./ -RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock - -# Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y -RUN sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json - -# Disable announcement plugin of jupyterlab -RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements" - -# Fix permissions to support pip in Openshift environments -RUN chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \ +RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock && \ + # Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y \ + sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json && \ + # Disable announcement plugin of jupyterlab \ + jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \ + # Fix permissions to support pip in Openshift environments \ + chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \ fix-permissions /opt/app-root -P diff --git a/runtimes/datascience/ubi8-python-3.8/utils/bootstrapper.py b/runtimes/datascience/ubi8-python-3.8/utils/bootstrapper.py index 6163cdd84..34a39cdcc 100644 --- a/runtimes/datascience/ubi8-python-3.8/utils/bootstrapper.py +++ b/runtimes/datascience/ubi8-python-3.8/utils/bootstrapper.py @@ -750,7 +750,6 @@ def main(): input_params = OpUtil.parse_arguments(sys.argv[1:]) OpUtil.log_operation_info("starting operation") t0 = time.time() - OpUtil.package_install(user_volume_path=input_params.get("user-volume-path")) # Create the appropriate instance, process dependencies and execute the operation file_op = FileOpBase.get_instance(**input_params) diff --git a/runtimes/datascience/ubi9-python-3.9/utils/bootstrapper.py b/runtimes/datascience/ubi9-python-3.9/utils/bootstrapper.py index 6163cdd84..34a39cdcc 100644 --- a/runtimes/datascience/ubi9-python-3.9/utils/bootstrapper.py +++ b/runtimes/datascience/ubi9-python-3.9/utils/bootstrapper.py @@ -750,7 +750,6 @@ def main(): input_params = OpUtil.parse_arguments(sys.argv[1:]) OpUtil.log_operation_info("starting operation") t0 = time.time() - OpUtil.package_install(user_volume_path=input_params.get("user-volume-path")) # Create the appropriate instance, process dependencies and execute the operation file_op = FileOpBase.get_instance(**input_params) diff --git a/runtimes/minimal/ubi8-python-3.8/utils/bootstrapper.py b/runtimes/minimal/ubi8-python-3.8/utils/bootstrapper.py index 6163cdd84..34a39cdcc 100644 --- a/runtimes/minimal/ubi8-python-3.8/utils/bootstrapper.py +++ b/runtimes/minimal/ubi8-python-3.8/utils/bootstrapper.py @@ -750,7 +750,6 @@ def main(): input_params = OpUtil.parse_arguments(sys.argv[1:]) OpUtil.log_operation_info("starting operation") t0 = time.time() - OpUtil.package_install(user_volume_path=input_params.get("user-volume-path")) # Create the appropriate instance, process dependencies and execute the operation file_op = FileOpBase.get_instance(**input_params) diff --git a/runtimes/minimal/ubi9-python-3.9/utils/bootstrapper.py b/runtimes/minimal/ubi9-python-3.9/utils/bootstrapper.py index 6163cdd84..34a39cdcc 100644 --- a/runtimes/minimal/ubi9-python-3.9/utils/bootstrapper.py +++ b/runtimes/minimal/ubi9-python-3.9/utils/bootstrapper.py @@ -750,7 +750,6 @@ def main(): input_params = OpUtil.parse_arguments(sys.argv[1:]) OpUtil.log_operation_info("starting operation") t0 = time.time() - OpUtil.package_install(user_volume_path=input_params.get("user-volume-path")) # Create the appropriate instance, process dependencies and execute the operation file_op = FileOpBase.get_instance(**input_params) diff --git a/runtimes/pytorch/ubi8-python-3.8/utils/bootstrapper.py b/runtimes/pytorch/ubi8-python-3.8/utils/bootstrapper.py index 6163cdd84..34a39cdcc 100644 --- a/runtimes/pytorch/ubi8-python-3.8/utils/bootstrapper.py +++ b/runtimes/pytorch/ubi8-python-3.8/utils/bootstrapper.py @@ -750,7 +750,6 @@ def main(): input_params = OpUtil.parse_arguments(sys.argv[1:]) OpUtil.log_operation_info("starting operation") t0 = time.time() - OpUtil.package_install(user_volume_path=input_params.get("user-volume-path")) # Create the appropriate instance, process dependencies and execute the operation file_op = FileOpBase.get_instance(**input_params) diff --git a/runtimes/pytorch/ubi9-python-3.9/utils/bootstrapper.py b/runtimes/pytorch/ubi9-python-3.9/utils/bootstrapper.py index 6163cdd84..34a39cdcc 100644 --- a/runtimes/pytorch/ubi9-python-3.9/utils/bootstrapper.py +++ b/runtimes/pytorch/ubi9-python-3.9/utils/bootstrapper.py @@ -750,7 +750,6 @@ def main(): input_params = OpUtil.parse_arguments(sys.argv[1:]) OpUtil.log_operation_info("starting operation") t0 = time.time() - OpUtil.package_install(user_volume_path=input_params.get("user-volume-path")) # Create the appropriate instance, process dependencies and execute the operation file_op = FileOpBase.get_instance(**input_params) diff --git a/runtimes/rocm-pytorch/ubi9-python-3.9/Dockerfile b/runtimes/rocm-pytorch/ubi9-python-3.9/Dockerfile index cd1114cd9..e1cb03c14 100644 --- a/runtimes/rocm-pytorch/ubi9-python-3.9/Dockerfile +++ b/runtimes/rocm-pytorch/ubi9-python-3.9/Dockerfile @@ -17,10 +17,15 @@ WORKDIR /opt/app-root/bin COPY Pipfile.lock ./ # Copy Elyra dependencies for air-gapped enviroment COPY utils ./utils/ +# Copy utility script +COPY de-vendor-torch.sh ./ RUN echo "Installing softwares and packages" && \ micropipenv install && \ rm -f ./Pipfile.lock && \ + # De-vendor the ROCm libs that are embedded in Pytorch \ + ./de-vendor-torch.sh && \ + rm ./de-vendor-torch.sh && \ # Fix permissions to support pip in Openshift environments \ chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \ fix-permissions /opt/app-root -P diff --git a/runtimes/rocm-pytorch/ubi9-python-3.9/de-vendor-torch.sh b/runtimes/rocm-pytorch/ubi9-python-3.9/de-vendor-torch.sh new file mode 100755 index 000000000..780f8815e --- /dev/null +++ b/runtimes/rocm-pytorch/ubi9-python-3.9/de-vendor-torch.sh @@ -0,0 +1,41 @@ +#!/bin/sh +set -ex +# Replace PyTorch's vendored shared libraries with system libraries +# The script assumes that PyTorch is built with the same ROCm ABI as the +# system installation of ROCm. + +# Source: https://github.com/tiran/instructlab-containers/blob/main/containers/rocm/de-vendor-torch.sh + +PYTHON=python3.9 +ROCMLIB=/opt/rocm/lib +TORCHLIB=/opt/app-root/lib/${PYTHON}/site-packages/torch/lib + +ln -sf /usr/lib64/libdrm.so.2 ${TORCHLIB}/libdrm.so +ln -sf /usr/lib64/libdrm_amdgpu.so.1 ${TORCHLIB}/libdrm_amdgpu.so + +ln -sf ${ROCMLIB}/libamd_comgr.so.2 ${TORCHLIB}/libamd_comgr.so +ln -sf ${ROCMLIB}/libamdhip64.so.6 ${TORCHLIB}/libamdhip64.so +ln -sf ${ROCMLIB}/libhipblaslt.so.0 ${TORCHLIB}/libhipblaslt.so +ln -sf ${ROCMLIB}/libhipblas.so.2 ${TORCHLIB}/libhipblas.so +ln -sf ${ROCMLIB}/libhipfft.so.0 ${TORCHLIB}/libhipfft.so +ln -sf ${ROCMLIB}/libhiprand.so.1 ${TORCHLIB}/libhiprand.so +ln -sf ${ROCMLIB}/libhiprtc.so.6 ${TORCHLIB}/libhiprtc.so +ln -sf ${ROCMLIB}/libhipsolver.so.0 ${TORCHLIB}/libhipsolver.so +ln -sf ${ROCMLIB}/libhipsparse.so.1 ${TORCHLIB}/libhipsparse.so +ln -sf ${ROCMLIB}/libhsa-runtime64.so.1 ${TORCHLIB}/libhsa-runtime64.so +ln -sf ${ROCMLIB}/libMIOpen.so.1 ${TORCHLIB}/libMIOpen.so +ln -sf ${ROCMLIB}/librccl.so.1 ${TORCHLIB}/librccl.so +ln -sf ${ROCMLIB}/librocblas.so.4 ${TORCHLIB}/librocblas.so +ln -sf ${ROCMLIB}/librocfft.so.0 ${TORCHLIB}/librocfft.so +ln -sf ${ROCMLIB}/librocm_smi64.so.6 ${TORCHLIB}/librocm_smi64.so +ln -sf ${ROCMLIB}/librocrand.so.1 ${TORCHLIB}/librocrand.so +ln -sf ${ROCMLIB}/librocsolver.so.0 ${TORCHLIB}/librocsolver.so +ln -sf ${ROCMLIB}/librocsparse.so.1 ${TORCHLIB}/librocsparse.so +ln -sf ${ROCMLIB}/libroctracer64.so.4 ${TORCHLIB}/libroctracer64.so +ln -sf ${ROCMLIB}/libroctx64.so.4 ${TORCHLIB}/libroctx64.so + +rm -rf ${TORCHLIB}/rocblas +ln -sf ${ROCMLIB}/rocblas ${TORCHLIB}/rocblas + +rm -rf ${TORCHLIB}/hipblaslt +ln -sf ${ROCMLIB}/hipblaslt ${TORCHLIB}/hipblaslt diff --git a/runtimes/rocm-pytorch/ubi9-python-3.9/utils/bootstrapper.py b/runtimes/rocm-pytorch/ubi9-python-3.9/utils/bootstrapper.py index 6163cdd84..34a39cdcc 100644 --- a/runtimes/rocm-pytorch/ubi9-python-3.9/utils/bootstrapper.py +++ b/runtimes/rocm-pytorch/ubi9-python-3.9/utils/bootstrapper.py @@ -750,7 +750,6 @@ def main(): input_params = OpUtil.parse_arguments(sys.argv[1:]) OpUtil.log_operation_info("starting operation") t0 = time.time() - OpUtil.package_install(user_volume_path=input_params.get("user-volume-path")) # Create the appropriate instance, process dependencies and execute the operation file_op = FileOpBase.get_instance(**input_params) diff --git a/runtimes/rocm-tensorflow/ubi9-python-3.9/utils/bootstrapper.py b/runtimes/rocm-tensorflow/ubi9-python-3.9/utils/bootstrapper.py index 6163cdd84..34a39cdcc 100644 --- a/runtimes/rocm-tensorflow/ubi9-python-3.9/utils/bootstrapper.py +++ b/runtimes/rocm-tensorflow/ubi9-python-3.9/utils/bootstrapper.py @@ -750,7 +750,6 @@ def main(): input_params = OpUtil.parse_arguments(sys.argv[1:]) OpUtil.log_operation_info("starting operation") t0 = time.time() - OpUtil.package_install(user_volume_path=input_params.get("user-volume-path")) # Create the appropriate instance, process dependencies and execute the operation file_op = FileOpBase.get_instance(**input_params) diff --git a/runtimes/tensorflow/ubi8-python-3.8/utils/bootstrapper.py b/runtimes/tensorflow/ubi8-python-3.8/utils/bootstrapper.py index 6163cdd84..34a39cdcc 100644 --- a/runtimes/tensorflow/ubi8-python-3.8/utils/bootstrapper.py +++ b/runtimes/tensorflow/ubi8-python-3.8/utils/bootstrapper.py @@ -750,7 +750,6 @@ def main(): input_params = OpUtil.parse_arguments(sys.argv[1:]) OpUtil.log_operation_info("starting operation") t0 = time.time() - OpUtil.package_install(user_volume_path=input_params.get("user-volume-path")) # Create the appropriate instance, process dependencies and execute the operation file_op = FileOpBase.get_instance(**input_params) diff --git a/runtimes/tensorflow/ubi9-python-3.9/utils/bootstrapper.py b/runtimes/tensorflow/ubi9-python-3.9/utils/bootstrapper.py index 6163cdd84..34a39cdcc 100644 --- a/runtimes/tensorflow/ubi9-python-3.9/utils/bootstrapper.py +++ b/runtimes/tensorflow/ubi9-python-3.9/utils/bootstrapper.py @@ -750,7 +750,6 @@ def main(): input_params = OpUtil.parse_arguments(sys.argv[1:]) OpUtil.log_operation_info("starting operation") t0 = time.time() - OpUtil.package_install(user_volume_path=input_params.get("user-volume-path")) # Create the appropriate instance, process dependencies and execute the operation file_op = FileOpBase.get_instance(**input_params) diff --git a/tests/test_main.py b/tests/test_main.py index 609e7f810..892a80dde 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,6 +1,9 @@ from __future__ import annotations +import os +import logging import pathlib +import subprocess import tomllib from typing import TYPE_CHECKING @@ -20,3 +23,48 @@ def test_image_pipfiles(subtests: pytest_subtests.plugin.SubTests): pipfile = tomllib.load(fp) assert "requires" in pipfile, "Pipfile is missing a [[requires]] section" assert pipfile["requires"]["python_version"] == python, "Pipfile does not declare the expected Python version" + + +def test_files_that_should_be_same_are_same(subtests: pytest_subtests.plugin.SubTests): + file_groups = { + "ROCm de-vendor script": + [PROJECT_ROOT / "jupyter/rocm/pytorch/ubi9-python-3.9/de-vendor-torch.sh", + PROJECT_ROOT / "runtimes/rocm-pytorch/ubi9-python-3.9/de-vendor-torch.sh"] + } + for group_name, (first_file, *rest) in file_groups.items(): + with subtests.test(msg=f"Checking {group_name}"): + for file in rest: + assert first_file.read_text() == file.read_text(), f"The files {first_file} and {file} do not match" + + +def test_make_building_only_specified_images(subtests: pytest_subtests.plugin.SubTests): + for goals in (["rocm-jupyter-tensorflow-ubi9-python-3.9"], ["rocm-jupyter-tensorflow-ubi9-python-3.9", "base-ubi9-python-3.9"]): + with subtests.test(msg="Running goals", goals=goals): + lines = dryrun_make(goals, env={"BUILD_DEPENDENT_IMAGES": "no"}) + builds_number = 0 + for line in lines: + if "podman build" in line: + builds_number += 1 + assert builds_number == len(goals) + + +def test_make_disable_pushing(): + lines = dryrun_make(["rocm-jupyter-tensorflow-ubi9-python-3.9"], env={"PUSH_IMAGES": ""}) + for line in lines: + assert "podman push" not in line + + +def dryrun_make(make_args: list[str], env: dict[str, str] | None = None) -> list[str]: + env = env or {} + + try: + logging.info(f"Running make in --just-print mode for target(s) {make_args} with env {env}") + lines = subprocess.check_output(["make", "--just-print", *make_args], encoding="utf-8", + env={**os.environ, **env}, + cwd=PROJECT_ROOT).splitlines() + for line in lines: + logging.debug(line) + return lines + except subprocess.CalledProcessError as e: + print(e.stderr, e.stdout) + raise