[WIP] RHOAIENG-8780: Add dependency versions tests

.github/workflows/code-quality.yaml

@@ -55,8 +55,6 @@ jobs:
           ret_code=0
           echo "-- Checking a regular '*.json' files"
           for f in **/*.json; do echo "Checking: '${f}"; echo -n "  > "; cat $f | json_verify || ret_code=1; done
-          echo "-- Checking a 'Pipfile.lock' files"
-          for f in **/Pipfile.lock; do echo "Checking: '${f}"; echo -n "  > "; cat $f | json_verify || ret_code=1; done
           echo "-- Checking a '*.ipynb' Jupyter notebook files"
           for f in **/*.ipynb; do echo "Checking: '${f}"; echo -n "  > "; cat $f | json_verify || ret_code=1; done
           if test "${ret_code}" -ne 0; then

Makefile

@@ -15,7 +15,7 @@ endif
 IMAGE_TAG		 ?= $(RELEASE)_$(DATE)
 KUBECTL_BIN      ?= bin/kubectl
 KUBECTL_VERSION  ?= v1.23.11
-NOTEBOOK_REPO_BRANCH_BASE ?= https://raw.githubusercontent.com/opendatahub-io/notebooks/main
+NOTEBOOK_REPO_BRANCH_BASE ?= https://raw.githubusercontent.com/paulovmr/notebooks/notebooks-tests
 REQUIRED_RUNTIME_IMAGE_COMMANDS="curl python3"
 REQUIRED_CODE_SERVER_IMAGE_COMMANDS="curl python oc code-server"
 REQUIRED_R_STUDIO_IMAGE_COMMANDS="curl python oc /usr/lib/rstudio-server/bin/rserver"
@@ -394,8 +394,29 @@ undeploy-c9s-%-c9s-python-3.9: bin/kubectl
 #   ARG 1: UBI flavor
 #   ARG 1: Python kernel
 define test_with_papermill
+	$(KUBECTL_BIN) exec $(FULL_NOTEBOOK_NAME) -- /bin/sh -c "python3 -m pip install papermill toml" ; \
+	ls -la ; \
+	$(call test_with_papermill_common,$(1),$(2),$(3)) ; \
+	$(call test_with_papermill_specific,$(1),$(2),$(3))
+endef
+
+define test_with_papermill_common
+	$(eval PREFIX_NAME := $(subst /,-,$(1)_$(2))) \
+	$(KUBECTL_BIN) exec $(FULL_NOTEBOOK_NAME) -- /bin/sh -c "wget ${NOTEBOOK_REPO_BRANCH_BASE}/jupyter/common/test/test_notebook_common.ipynb -O test_notebook_common.ipynb && python3 -m papermill test_notebook_common.ipynb $(PREFIX_NAME)_common_output.ipynb --kernel python3 --stderr-file $(PREFIX_NAME)_common_error.txt" ; \
+    if [ $$? -ne 0 ]; then \
+		echo "ERROR: The $(1) $(2) notebook common tests encountered a failure. To investigate the issue, you can review the logs located in the ocp-ci cluster on 'artifacts/notebooks-e2e-tests/jupyter-$(1)-$(2)-$(3)-test-e2e' directory or run 'cat $(PREFIX_NAME)_common_error.txt' within your container. The make process has been aborted." ; \
+		exit 1 ; \
+	fi ; \
+	$(KUBECTL_BIN) exec $(FULL_NOTEBOOK_NAME) -- /bin/sh -c "cat $(PREFIX_NAME)_common_error.txt | grep --quiet FAILED" ; \
+	if [ $$? -eq 0 ]; then \
+		echo "ERROR: The $(1) $(2) notebook common tests encountered a failure. The make process has been aborted." ; \
+		$(KUBECTL_BIN) exec $(FULL_NOTEBOOK_NAME) -- /bin/sh -c "cat $(PREFIX_NAME)_common_error.txt" ; \
+		exit 1 ; \
+	fi
+endef
+
+define test_with_papermill_specific
 	$(eval PREFIX_NAME := $(subst /,-,$(1)_$(2))) \
-	$(KUBECTL_BIN) exec $(FULL_NOTEBOOK_NAME) -- /bin/sh -c "python3 -m pip install papermill" ; \
 	$(KUBECTL_BIN) exec $(FULL_NOTEBOOK_NAME) -- /bin/sh -c "wget ${NOTEBOOK_REPO_BRANCH_BASE}/jupyter/$(1)/$(2)-$(3)/test/test_notebook.ipynb -O test_notebook.ipynb && python3 -m papermill test_notebook.ipynb $(PREFIX_NAME)_output.ipynb --kernel python3 --stderr-file $(PREFIX_NAME)_error.txt" ; \
     if [ $$? -ne 0 ]; then \
 		echo "ERROR: The $(1) $(2) notebook encountered a failure. To investigate the issue, you can review the logs located in the ocp-ci cluster on 'artifacts/notebooks-e2e-tests/jupyter-$(1)-$(2)-$(3)-test-e2e' directory or run 'cat $(PREFIX_NAME)_error.txt' within your container. The make process has been aborted." ; \

base/c9s-python-3.9/.python-version

@@ -0,0 +1 @@
+3.9

base/c9s-python-3.9/Dockerfile

@@ -12,11 +12,11 @@ LABEL name="odh-notebook-base-centos-stream9-python-3.9" \
 
 WORKDIR /opt/app-root/bin
 
-# Install micropipenv to deploy packages from Pipfile.lock
-RUN pip install --no-cache-dir -U "micropipenv[toml]"
+# Install micropipenv to deploy packages from pyproject.toml and poetry.lock
+RUN pip install --no-cache-dir -U "micropipenv[toml]==1.7.0"
 
-# Install Python dependencies from Pipfile.lock file
-COPY Pipfile.lock ./
+# Install Python dependencies from pyproject.toml and poetry.lock file
+COPY pyproject.toml poetry.lock ./
 
 # OS Packages needs to be installed as root
 USER root
@@ -27,7 +27,7 @@ RUN dnf install -y mesa-libGL && dnf clean all && rm -rf /var/cache/yum
 # Other apps and tools installed as default user
 USER 1001
 
-RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock && \
+RUN echo "Installing softwares and packages" && micropipenv install --method poetry && rm -f ./pyproject.toml ./poetry.lock && \
     # Install the oc client \
     curl -L https://mirror.openshift.com/pub/openshift-v4/$(uname -m)/clients/ocp/stable/openshift-client-linux.tar.gz \
         -o /tmp/openshift-client-linux.tar.gz && \

base/c9s-python-3.9/pyproject.toml

@@ -0,0 +1,15 @@
+[tool.poetry]
+package-mode = false
+
+[tool.poetry.dependencies]
+python = ">=3.9.0,<3.10.0"
+wheel = "~=0.43.0"
+setuptools = "~=69.2.0"
+
+
+[tool.poetry.group.dev.dependencies]
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+

base/ubi8-python-3.8/.python-version

@@ -0,0 +1 @@
+3.8

base/ubi8-python-3.8/Dockerfile

@@ -12,13 +12,13 @@ LABEL name="odh-notebook-base-ubi8-python-3.8" \
 
 WORKDIR /opt/app-root/bin
 
-# Install micropipenv to deploy packages from Pipfile.lock
-RUN pip install --no-cache-dir -U "micropipenv[toml]"
+# Install micropipenv to deploy packages from pyproject.toml and poetry.lock
+RUN pip install --no-cache-dir -U "micropipenv[toml]==1.7.0"
 
-# Install Python dependencies from Pipfile.lock file
-COPY Pipfile.lock ./
+# Install Python dependencies from pyproject.toml and poetry.lock file
+COPY pyproject.toml poetry.lock ./
 
-RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock
+RUN echo "Installing softwares and packages" && micropipenv install --method poetry && rm -f ./pyproject.toml ./poetry.lock
 
 # OS Packages needs to be installed as root
 USER root

base/ubi8-python-3.8/pyproject.toml

@@ -0,0 +1,15 @@
+[tool.poetry]
+package-mode = false
+
+[tool.poetry.dependencies]
+python = ">=3.8.1,<3.9.0"
+wheel = "~=0.43.0"
+setuptools = "~=69.2.0"
+
+
+[tool.poetry.group.dev.dependencies]
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+

base/ubi9-python-3.9/.python-version

@@ -0,0 +1 @@
+3.9

base/ubi9-python-3.9/Dockerfile

@@ -12,13 +12,13 @@ LABEL name="odh-notebook-base-ubi9-python-3.9" \
 
 WORKDIR /opt/app-root/bin
 
-# Install micropipenv to deploy packages from Pipfile.lock
-RUN pip install --no-cache-dir -U "micropipenv[toml]"
+# Install micropipenv to deploy packages from pyproject.toml and poetry.lock
+RUN pip install --no-cache-dir -U "micropipenv[toml]==1.7.0"
 
-# Install Python dependencies from Pipfile.lock file
-COPY Pipfile.lock ./
+# Install Python dependencies from pyproject.toml and poetry.lock file
+COPY pyproject.toml poetry.lock ./
 
-RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock
+RUN echo "Installing softwares and packages" && micropipenv install --method poetry && rm -f ./pyproject.toml ./poetry.lock
 
 # OS Packages needs to be installed as root
 USER root