From 6e21ada228d7352e36db81cf52a4075eecec5e6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Fri, 31 May 2024 09:36:57 +0200 Subject: [PATCH] squash-me, tag images with hash, not date, and handle PR builds differently --- .../workflows/build-notebooks-TEMPLATE.yaml | 21 ++- .github/workflows/build-notebooks.yaml | 126 ++++++++++++------ ci/gen_gha_matrix_jobs.py | 1 + 3 files changed, 105 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build-notebooks-TEMPLATE.yaml b/.github/workflows/build-notebooks-TEMPLATE.yaml index e8700b51d..752a08687 100644 --- a/.github/workflows/build-notebooks-TEMPLATE.yaml +++ b/.github/workflows/build-notebooks-TEMPLATE.yaml @@ -5,10 +5,16 @@ name: Build & Publish Notebook Servers (TEMPLATE) "on": workflow_call: inputs: + # https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables + # https://docs.github.com/en/actions/learn-github-actions/contexts target: required: true description: "make target to build" type: string + github: + required: true + description: "top workflow's `github`" + type: string jobs: build: @@ -54,10 +60,23 @@ jobs: # podman bug? need to create this _after_ doing the reset mkdir -p /mnt/containers/tmp - - run: "make ${{ inputs.target }}" + # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push + - name: "push: make ${{ inputs.target }}" + run: "make ${{ inputs.target }}" + if: "${{ fromJson(inputs.github).event_name == 'push' }}" env: + IMAGE_TAG: "${{ github.ref_name }}_${{ github.sha }}" IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images" CACHE: "ghcr.io/${{ github.repository }}/workbench-images/build-cache" + # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request + - name: "pull_request: make ${{ inputs.target }}" + run: "make ${{ inputs.target }}" + if: "${{ fromJson(inputs.github).event_name == 'pull_request' }}" + env: + IMAGE_TAG: "${{ github.ref_name }}_${{ github.sha }}" + IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images/pr${{ fromJson(inputs.github).event.number }}" + CACHE: "ghcr.io/${{ github.repository }}/workbench-images/build-cache" + - run: df -h if: "${{ !cancelled() }}" diff --git a/.github/workflows/build-notebooks.yaml b/.github/workflows/build-notebooks.yaml index 79a3b767f..47c211e12 100644 --- a/.github/workflows/build-notebooks.yaml +++ b/.github/workflows/build-notebooks.yaml @@ -15,7 +15,8 @@ "needs": [], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "base-ubi8-python-3.8" + "target": "base-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -25,7 +26,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "jupyter-minimal-ubi8-python-3.8" + "target": "jupyter-minimal-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -35,7 +37,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "jupyter-datascience-ubi8-python-3.8" + "target": "jupyter-datascience-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -45,7 +48,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "cuda-ubi8-python-3.8" + "target": "cuda-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -55,7 +59,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "cuda-jupyter-minimal-ubi8-python-3.8" + "target": "cuda-jupyter-minimal-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -65,7 +70,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "cuda-jupyter-datascience-ubi8-python-3.8" + "target": "cuda-jupyter-datascience-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -75,7 +81,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "jupyter-trustyai-ubi8-python-3.8" + "target": "jupyter-trustyai-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -85,7 +92,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "habana-jupyter-1.9.0-ubi8-python-3.8" + "target": "habana-jupyter-1.9.0-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -95,7 +103,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "habana-jupyter-1.10.0-ubi8-python-3.8" + "target": "habana-jupyter-1.10.0-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -105,7 +114,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "habana-jupyter-1.11.0-ubi8-python-3.8" + "target": "habana-jupyter-1.11.0-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -115,7 +125,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "habana-jupyter-1.13.0-ubi8-python-3.8" + "target": "habana-jupyter-1.13.0-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -125,7 +136,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "runtime-minimal-ubi8-python-3.8" + "target": "runtime-minimal-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -135,7 +147,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "runtime-datascience-ubi8-python-3.8" + "target": "runtime-datascience-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -145,7 +158,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "runtime-pytorch-ubi8-python-3.8" + "target": "runtime-pytorch-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -155,7 +169,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "runtime-cuda-tensorflow-ubi8-python-3.8" + "target": "runtime-cuda-tensorflow-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -163,7 +178,8 @@ "needs": [], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "base-ubi9-python-3.9" + "target": "base-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -173,7 +189,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "jupyter-minimal-ubi9-python-3.9" + "target": "jupyter-minimal-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -183,7 +200,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "jupyter-datascience-ubi9-python-3.9" + "target": "jupyter-datascience-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -193,7 +211,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "cuda-ubi9-python-3.9" + "target": "cuda-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -203,7 +222,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "cuda-jupyter-minimal-ubi9-python-3.9" + "target": "cuda-jupyter-minimal-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -213,7 +233,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "cuda-jupyter-datascience-ubi9-python-3.9" + "target": "cuda-jupyter-datascience-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -223,7 +244,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "cuda-jupyter-tensorflow-ubi9-python-3.9" + "target": "cuda-jupyter-tensorflow-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -233,7 +255,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "jupyter-pytorch-ubi9-python-3.9" + "target": "jupyter-pytorch-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -243,7 +266,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "jupyter-trustyai-ubi9-python-3.9" + "target": "jupyter-trustyai-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -253,7 +277,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "runtime-minimal-ubi9-python-3.9" + "target": "runtime-minimal-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -263,7 +288,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "runtime-datascience-ubi9-python-3.9" + "target": "runtime-datascience-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -273,7 +299,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "runtime-pytorch-ubi9-python-3.9" + "target": "runtime-pytorch-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -283,7 +310,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "runtime-cuda-tensorflow-ubi9-python-3.9" + "target": "runtime-cuda-tensorflow-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -293,7 +321,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "codeserver-ubi9-python-3.9" + "target": "codeserver-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -303,7 +332,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "intel-base-gpu-ubi9-python-3.9" + "target": "intel-base-gpu-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -313,7 +343,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "intel-runtime-tensorflow-ubi9-python-3.9" + "target": "intel-runtime-tensorflow-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -323,7 +354,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "jupyter-intel-tensorflow-ubi9-python-3.9" + "target": "jupyter-intel-tensorflow-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -333,7 +365,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "intel-runtime-pytorch-ubi9-python-3.9" + "target": "intel-runtime-pytorch-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -343,7 +376,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "jupyter-intel-pytorch-ubi9-python-3.9" + "target": "jupyter-intel-pytorch-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -353,7 +387,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "intel-runtime-ml-ubi9-python-3.9" + "target": "intel-runtime-ml-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -363,7 +398,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "jupyter-intel-ml-ubi9-python-3.9" + "target": "jupyter-intel-ml-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -371,7 +407,8 @@ "needs": [], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "base-c9s-python-3.9" + "target": "base-c9s-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -381,7 +418,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "cuda-c9s-python-3.9" + "target": "cuda-c9s-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -391,7 +429,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "rstudio-c9s-python-3.9" + "target": "rstudio-c9s-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -401,7 +440,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "cuda-rstudio-c9s-python-3.9" + "target": "cuda-rstudio-c9s-python-3.9", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -409,7 +449,8 @@ "needs": [], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "base-anaconda-python-3.8" + "target": "base-anaconda-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" }, @@ -419,7 +460,8 @@ ], "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { - "target": "jupyter-datascience-anaconda-python-3.8" + "target": "jupyter-datascience-anaconda-python-3.8", + "github": "${{ toJSON(github) }}" }, "secrets": "inherit" } diff --git a/ci/gen_gha_matrix_jobs.py b/ci/gen_gha_matrix_jobs.py index d1e4e802f..4bcdd8d4f 100644 --- a/ci/gen_gha_matrix_jobs.py +++ b/ci/gen_gha_matrix_jobs.py @@ -103,6 +103,7 @@ def write_github_workflow_file(tree: dict[str, list[str]], path: pathlib.Path) - "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", "with": { "target": task, + "github": "${{ toJSON(github) }}", }, "secrets": "inherit", }