Merge pull request #289 from harshad16/rsync-2024a

[release-2024a] sync 2024a branch with upstream opendatahub 2024a branch
red-hat-data-services · Jun 28, 2024 · 3ccfb36 · 3ccfb36
2 parents 391e435 + 85ccf1a
commit 3ccfb36
Show file tree

Hide file tree

Showing 69 changed files with 12,402 additions and 2,717 deletions.
diff --git a/.github/workflows/build-notebooks-TEMPLATE.yaml b/.github/workflows/build-notebooks-TEMPLATE.yaml
@@ -34,6 +34,38 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Free up additional disk space
+        # https://docs.github.com/en/actions/learn-github-actions/expressions
+        if: "${{ contains(inputs.target, 'amd') || contains(inputs.target, 'cuda') || contains(inputs.target, 'intel') ||
+         contains(inputs.target, 'pytorch') || contains(inputs.target, 'tensorflow') }}"
+        run: |
+          set -x
+
+          df -h
+
+          sudo rm -rf /usr/local/lib/android &
+          sudo rm -rf /usr/local/share/boost &
+          sudo rm -rf /usr/local/lib/node_modules &
+          sudo rm -rf /usr/share/dotnet &
+          sudo rm -rf /opt/ghc &
+          sudo rm -rf /opt/hostedtoolcache/CodeQL &
+
+          sudo docker image prune --all --force &
+
+          wait
+
+          df -h
+
+      - name: Mount lvm overlay for podman builds
+        run: |
+          df -h
+          free -h
+
+          bash ./ci/cached-builds/gha_lvm_overlay.bash
+
+          df -h
+          free -h
+
       # https://github.com/containers/buildah/issues/2521#issuecomment-884779112
       - name: Workaround https://github.com/containers/podman/issues/22152#issuecomment-2027705598
         run: sudo apt-get -qq remove podman crun
@@ -58,12 +90,10 @@ jobs:
           mkdir -p $HOME/.config/containers/
           cp ci/cached-builds/containers.conf $HOME/.config/containers/containers.conf
           cp ci/cached-builds/storage.conf $HOME/.config/containers/storage.conf
+
           # should at least reset storage when touching storage.conf
-          sudo mkdir -p /mnt/containers/
-          sudo chown -R $USER:$USER /mnt/containers
           podman system reset --force
-          # podman bug? need to create this _after_ doing the reset
-          mkdir -p /mnt/containers/tmp
+          mkdir -p $HOME/.local/share/containers/storage/tmp
 
       # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push
       - name: "push: make ${{ inputs.target }}"

diff --git a/.github/workflows/build-notebooks-pr.yaml b/.github/workflows/build-notebooks-pr.yaml
@@ -1,28 +1,40 @@
 ---
 "name": "Build Notebooks"
-"permissions":
-  "packages": "read"
 "on":
   "pull_request":
 
+permissions:
+  contents: read
+  packages: read
+  pull-requests: read
+
 jobs:
   gen:
     name: Generate job matrix
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.gen.outputs.matrix }}
+      has_jobs: ${{ steps.gen.outputs.has_jobs }}
     steps:
       - uses: actions/checkout@v4
-      - run: python3 ci/cached-builds/gen_gha_matrix_jobs.py
+
+      - run: |
+          python3 ci/cached-builds/gen_gha_matrix_jobs.py \
+            --owner=${{ github.repository_owner }} \
+            --repo=${{ github.event.pull_request.base.repo.name }} \
+            --pr-number=${{ github.event.pull_request.number }} \
+            --skip-unchanged
         id: gen
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  # base images
   build:
     needs: ["gen"]
     strategy:
       fail-fast: false
       matrix: "${{ fromJson(needs.gen.outputs.matrix) }}"
     uses: ./.github/workflows/build-notebooks-TEMPLATE.yaml
+    if: ${{ fromJson(needs.gen.outputs.has_jobs) }}
     with:
       target: "${{ matrix.target }}"
       github: "${{ toJSON(github) }}"

diff --git a/.github/workflows/build-notebooks.yaml b/.github/workflows/build-notebooks.yaml
@@ -444,6 +444,61 @@
             },
             "secrets": "inherit"
         },
+        "amd-c9s-python-3_9": {
+            "needs": [
+                "base-c9s-python-3_9"
+            ],
+            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
+            "with": {
+                "target": "amd-c9s-python-3.9",
+                "github": "${{ toJSON(github) }}"
+            },
+            "secrets": "inherit"
+        },
+        "amd-jupyter-minimal-c9s-python-3_9": {
+            "needs": [
+                "amd-c9s-python-3_9"
+            ],
+            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
+            "with": {
+                "target": "amd-jupyter-minimal-c9s-python-3.9",
+                "github": "${{ toJSON(github) }}"
+            },
+            "secrets": "inherit"
+        },
+        "amd-jupyter-datascience-c9s-python-3_9": {
+            "needs": [
+                "amd-jupyter-minimal-c9s-python-3_9"
+            ],
+            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
+            "with": {
+                "target": "amd-jupyter-datascience-c9s-python-3.9",
+                "github": "${{ toJSON(github) }}"
+            },
+            "secrets": "inherit"
+        },
+        "amd-jupyter-tensorflow-c9s-python-3_9": {
+            "needs": [
+                "amd-jupyter-datascience-c9s-python-3_9"
+            ],
+            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
+            "with": {
+                "target": "amd-jupyter-tensorflow-c9s-python-3.9",
+                "github": "${{ toJSON(github) }}"
+            },
+            "secrets": "inherit"
+        },
+        "amd-jupyter-pytorch-c9s-python-3_9": {
+            "needs": [
+                "amd-jupyter-datascience-c9s-python-3_9"
+            ],
+            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
+            "with": {
+                "target": "amd-jupyter-pytorch-c9s-python-3.9",
+                "github": "${{ toJSON(github) }}"
+            },
+            "secrets": "inherit"
+        },
         "base-anaconda-python-3_8": {
             "needs": [],
             "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",

diff --git a/.github/workflows/params-env.yaml b/.github/workflows/params-env.yaml
@@ -1,11 +1,13 @@
 ---
 name: Validation of image references (image SHAs) in params.env and runtime images
 on:  # yamllint disable-line rule:truthy
+  push:
   pull_request:
     paths:
       - 'manifests/base/commit.env'
       - 'manifests/base/params.env'
       - 'ci/check-params-env.sh'
+  workflow_dispatch:
 
 permissions:
   contents: read

diff --git a/.github/workflows/runtimes-digest-updater-upstream.yaml b/.github/workflows/runtimes-digest-updater-upstream.yaml
diff --git a/.github/workflows/sync-branches-through-pr.yml b/.github/workflows/sync-branches-through-pr.yml
@@ -0,0 +1,47 @@
+---
+name: Sync branches through Pull Request
+
+on:  # yamllint disable-line rule:truthy
+  workflow_dispatch:
+    inputs:
+      source:
+        description: Source branch
+        required: true
+      target:
+        description: Target branch
+        required: true
+
+jobs:
+  sync:
+    permissions:
+      contents: write
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.target }}
+          fetch-depth: 0
+
+      - name: Prepare sync branch
+        id: prepare
+        run: |
+          git fetch origin ${{ github.event.inputs.source }}
+          git reset --hard origin/${{ github.event.inputs.source }}
+
+          TIMESTAMP=$(date +'%Y%m%d%H%M%S')
+          SYNC_BRANCH=sync__${{ github.event.inputs.source }}__${{ github.event.inputs.target }}__${TIMESTAMP}
+          echo "branch=$SYNC_BRANCH" >> $GITHUB_OUTPUT
+
+      - name: Create pull request
+        uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c  # v6.1.0
+        with:
+          branch: ${{ steps.prepare.outputs.branch }}
+          title: "Sync `${{ github.event.inputs.target }}` branch with `${{ github.event.inputs.source }}` branch"
+          body: |
+            :robot: This is an automated Pull Request created by `/.github/workflows/sync-branches-through-pr.yml`.
+
+            It merges all commits from `${{ github.event.inputs.source }}` branch into `${{ github.event.inputs.target }}` branch.
+
+            :warning: **IMPORTANT NOTE**: Remember to delete the `${{ steps.prepare.outputs.branch }}` branch after merging the changes.
diff --git a/Makefile b/Makefile
@@ -59,6 +59,7 @@ endef
 #   ARG 2: Path of image context we want to build.
 #   ARG 3: Base image tag name (optional).
 define image
+	$(info #*# Image build directory: <$(2)> #(MACHINE-PARSED LINE)#*#...)
 	$(call build_image,$(1),$(2),$(3))
 	$(call push_image,$(1))
 endef
@@ -272,6 +273,31 @@ cuda-rstudio-c9s-python-3.9: cuda-c9s-python-3.9
 .PHONY: base-rhel9-python-3.9
 base-rhel9-python-3.9:
 	$(call image,$@,base/rhel9-python-3.9)
+####################################### Buildchain for AMD Python 3.9 using C9S #######################################
+.PHONY: amd-c9s-python-3.9
+amd-c9s-python-3.9: base-c9s-python-3.9	
+	$(call image,$@,amd/c9s-python-3.9,$<)
+
+# We are only using c9s base image here onwards, 
+# DON'T confuse due to the ubi9 mention, it just directory name.
+.PHONY: amd-jupyter-minimal-c9s-python-3.9
+amd-jupyter-minimal-c9s-python-3.9: amd-c9s-python-3.9
+	$(call image,$@,jupyter/minimal/ubi9-python-3.9,$<)
+
+# Build and push jupyter-datascience-ubi9-python-3.9 image to the registry
+.PHONY: amd-jupyter-datascience-c9s-python-3.9
+amd-jupyter-datascience-c9s-python-3.9: amd-jupyter-minimal-c9s-python-3.9
+	$(call image,$@,jupyter/datascience/ubi9-python-3.9,$<)
+
+# Build and push jupyter-tensorflow-ubi9-python-3.9 image to the registry
+.PHONY: amd-jupyter-tensorflow-c9s-python-3.9
+amd-jupyter-tensorflow-c9s-python-3.9: amd-jupyter-datascience-c9s-python-3.9
+	$(call image,$@,jupyter/amd/tensorflow/ubi9-python-3.9,$<)
+
+# Build and push jupyter-pytorch-ubi9-python-3.9 image to the registry
+.PHONY: amd-jupyter-pytorch-c9s-python-3.9
+amd-jupyter-pytorch-c9s-python-3.9: amd-jupyter-datascience-c9s-python-3.9
+	$(call image,$@,jupyter/amd/pytorch/ubi9-python-3.9,$<)
 
 ####################################### Buildchain for Anaconda Python #######################################