diff --git a/.github/workflows/build-notebooks-TEMPLATE.yaml b/.github/workflows/build-notebooks-TEMPLATE.yaml new file mode 100644 index 000000000..808e428d0 --- /dev/null +++ b/.github/workflows/build-notebooks-TEMPLATE.yaml @@ -0,0 +1,93 @@ +# inspired by +# https://github.com/thesuperzapper/kubeflow/blob/master/.github/workflows/example_notebook_servers_publish_TEMPLATE.yaml +--- +name: Build & Publish Notebook Servers (TEMPLATE) +"on": + workflow_call: + inputs: + # https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables + # https://docs.github.com/en/actions/learn-github-actions/contexts + target: + required: true + description: "make target to build" + type: string + github: + required: true + description: "top workflow's `github`" + type: string + +jobs: + build: + runs-on: ubuntu-latest + env: + # GitHub image registry used for storing $(CONTAINER_ENGINE)'s cache + CACHE: "ghcr.io/${{ github.repository }}/workbench-images/build-cache" + + steps: + + - uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # https://github.com/containers/buildah/issues/2521#issuecomment-884779112 + - name: Workaround https://github.com/containers/podman/issues/22152#issuecomment-2027705598 + run: sudo apt-get -qq remove podman crun + + - uses: actions/cache@v4 + id: cached-linuxbrew + with: + path: /home/linuxbrew/.linuxbrew + key: linuxbrew + + - name: Install podman + if: steps.cached-linuxbrew.outputs.cache-hit != 'true' + run: | + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + /home/linuxbrew/.linuxbrew/bin/brew install podman + + - name: Add linuxbrew to PATH + run: echo "/home/linuxbrew/.linuxbrew/bin/" >> $GITHUB_PATH + + - name: Configure Podman + run: | + mkdir -p $HOME/.config/containers/ + cp ci/cached-builds/containers.conf $HOME/.config/containers/containers.conf + cp ci/cached-builds/storage.conf $HOME/.config/containers/storage.conf + # should at least reset storage when touching storage.conf + sudo mkdir -p /mnt/containers/ + sudo chown -R $USER:$USER /mnt/containers + podman system reset --force + # podman bug? need to create this _after_ doing the reset + mkdir -p /mnt/containers/tmp + + # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push + - name: "push: make ${{ inputs.target }}" + run: "make ${{ inputs.target }}" + if: "${{ fromJson(inputs.github).event_name == 'push' }}" + env: + IMAGE_TAG: "${{ github.ref_name }}_${{ github.sha }}" + IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images" + CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }} --cache-to ${{ env.CACHE }}" + + # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request + - name: "pull_request: make ${{ inputs.target }}" + run: | + # start a black hole container registry as make target always does a push + mkdir -p $HOME/.config/containers/registries.conf.d/ + cp ci/cached-builds/insecure_localhost_registry.conf $HOME/.config/containers/registries.conf.d/insecure_localhost_registry.conf + go run ci/cached-builds/dev_null_container_registry.go & + # build and push the image + make ${{ inputs.target }} + if: "${{ fromJson(inputs.github).event_name == 'pull_request' }}" + env: + IMAGE_TAG: "${{ github.sha }}" + IMAGE_REGISTRY: "localhost:5000/workbench-images" + CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }}" + + - run: df -h + if: "${{ !cancelled() }}" diff --git a/.github/workflows/build-notebooks-pr.yaml b/.github/workflows/build-notebooks-pr.yaml new file mode 100644 index 000000000..d04d90a6c --- /dev/null +++ b/.github/workflows/build-notebooks-pr.yaml @@ -0,0 +1,29 @@ +--- +"name": "Build Notebooks" +"permissions": + "packages": "read" +"on": + "pull_request": + +jobs: + gen: + name: Generate job matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.gen.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - run: python3 ci/cached-builds/gen_gha_matrix_jobs.py + id: gen + + # base images + build: + needs: ["gen"] + strategy: + fail-fast: false + matrix: "${{ fromJson(needs.gen.outputs.matrix) }}" + uses: ./.github/workflows/build-notebooks-TEMPLATE.yaml + with: + target: "${{ matrix.target }}" + github: "${{ toJSON(github) }}" + secrets: inherit diff --git a/.github/workflows/build-notebooks.yaml b/.github/workflows/build-notebooks.yaml new file mode 100644 index 000000000..b76bd1fea --- /dev/null +++ b/.github/workflows/build-notebooks.yaml @@ -0,0 +1,468 @@ +--- +# This file is autogenerated by ci/cached-builds/gen_gha_matrix_jobs.py +{ + "name": "Build Notebooks", + "permissions": { + "packages": "write" + }, + "on": { + "push": {}, + "workflow_dispatch": {} + }, + "jobs": { + "base-ubi8-python-3_8": { + "needs": [], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "base-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "jupyter-minimal-ubi8-python-3_8": { + "needs": [ + "base-ubi8-python-3_8" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "jupyter-minimal-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "jupyter-datascience-ubi8-python-3_8": { + "needs": [ + "jupyter-minimal-ubi8-python-3_8" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "jupyter-datascience-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "cuda-ubi8-python-3_8": { + "needs": [ + "base-ubi8-python-3_8" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "cuda-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "cuda-jupyter-minimal-ubi8-python-3_8": { + "needs": [ + "cuda-ubi8-python-3_8" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "cuda-jupyter-minimal-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "cuda-jupyter-datascience-ubi8-python-3_8": { + "needs": [ + "cuda-jupyter-minimal-ubi8-python-3_8" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "cuda-jupyter-datascience-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "jupyter-trustyai-ubi8-python-3_8": { + "needs": [ + "jupyter-datascience-ubi8-python-3_8" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "jupyter-trustyai-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "habana-jupyter-1_9_0-ubi8-python-3_8": { + "needs": [ + "jupyter-datascience-ubi8-python-3_8" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "habana-jupyter-1.9.0-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "habana-jupyter-1_10_0-ubi8-python-3_8": { + "needs": [ + "jupyter-datascience-ubi8-python-3_8" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "habana-jupyter-1.10.0-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "habana-jupyter-1_11_0-ubi8-python-3_8": { + "needs": [ + "jupyter-datascience-ubi8-python-3_8" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "habana-jupyter-1.11.0-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "habana-jupyter-1_13_0-ubi8-python-3_8": { + "needs": [ + "jupyter-datascience-ubi8-python-3_8" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "habana-jupyter-1.13.0-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "runtime-minimal-ubi8-python-3_8": { + "needs": [ + "base-ubi8-python-3_8" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "runtime-minimal-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "runtime-datascience-ubi8-python-3_8": { + "needs": [ + "base-ubi8-python-3_8" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "runtime-datascience-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "runtime-pytorch-ubi8-python-3_8": { + "needs": [ + "base-ubi8-python-3_8" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "runtime-pytorch-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "runtime-cuda-tensorflow-ubi8-python-3_8": { + "needs": [ + "cuda-ubi8-python-3_8" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "runtime-cuda-tensorflow-ubi8-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "base-ubi9-python-3_9": { + "needs": [], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "base-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "jupyter-minimal-ubi9-python-3_9": { + "needs": [ + "base-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "jupyter-minimal-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "jupyter-datascience-ubi9-python-3_9": { + "needs": [ + "jupyter-minimal-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "jupyter-datascience-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "cuda-ubi9-python-3_9": { + "needs": [ + "base-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "cuda-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "cuda-jupyter-minimal-ubi9-python-3_9": { + "needs": [ + "cuda-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "cuda-jupyter-minimal-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "cuda-jupyter-datascience-ubi9-python-3_9": { + "needs": [ + "cuda-jupyter-minimal-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "cuda-jupyter-datascience-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "cuda-jupyter-tensorflow-ubi9-python-3_9": { + "needs": [ + "cuda-jupyter-datascience-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "cuda-jupyter-tensorflow-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "jupyter-pytorch-ubi9-python-3_9": { + "needs": [ + "cuda-jupyter-datascience-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "jupyter-pytorch-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "jupyter-trustyai-ubi9-python-3_9": { + "needs": [ + "jupyter-datascience-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "jupyter-trustyai-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "runtime-minimal-ubi9-python-3_9": { + "needs": [ + "base-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "runtime-minimal-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "runtime-datascience-ubi9-python-3_9": { + "needs": [ + "base-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "runtime-datascience-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "runtime-pytorch-ubi9-python-3_9": { + "needs": [ + "base-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "runtime-pytorch-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "runtime-cuda-tensorflow-ubi9-python-3_9": { + "needs": [ + "cuda-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "runtime-cuda-tensorflow-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "codeserver-ubi9-python-3_9": { + "needs": [ + "base-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "codeserver-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "intel-base-gpu-ubi9-python-3_9": { + "needs": [ + "base-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "intel-base-gpu-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "intel-runtime-tensorflow-ubi9-python-3_9": { + "needs": [ + "intel-base-gpu-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "intel-runtime-tensorflow-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "jupyter-intel-tensorflow-ubi9-python-3_9": { + "needs": [ + "intel-base-gpu-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "jupyter-intel-tensorflow-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "intel-runtime-pytorch-ubi9-python-3_9": { + "needs": [ + "intel-base-gpu-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "intel-runtime-pytorch-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "jupyter-intel-pytorch-ubi9-python-3_9": { + "needs": [ + "intel-base-gpu-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "jupyter-intel-pytorch-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "intel-runtime-ml-ubi9-python-3_9": { + "needs": [ + "base-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "intel-runtime-ml-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "jupyter-intel-ml-ubi9-python-3_9": { + "needs": [ + "base-ubi9-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "jupyter-intel-ml-ubi9-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "base-c9s-python-3_9": { + "needs": [], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "base-c9s-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "cuda-c9s-python-3_9": { + "needs": [ + "base-c9s-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "cuda-c9s-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "rstudio-c9s-python-3_9": { + "needs": [ + "base-c9s-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "rstudio-c9s-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "cuda-rstudio-c9s-python-3_9": { + "needs": [ + "cuda-c9s-python-3_9" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "cuda-rstudio-c9s-python-3.9", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "base-anaconda-python-3_8": { + "needs": [], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "base-anaconda-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + }, + "jupyter-datascience-anaconda-python-3_8": { + "needs": [ + "base-anaconda-python-3_8" + ], + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": "jupyter-datascience-anaconda-python-3.8", + "github": "${{ toJSON(github) }}" + }, + "secrets": "inherit" + } + } +} diff --git a/.github/workflows/code-quality.yaml b/.github/workflows/code-quality.yaml index c3cc98cc0..e1008aec2 100644 --- a/.github/workflows/code-quality.yaml +++ b/.github/workflows/code-quality.yaml @@ -9,6 +9,27 @@ permissions: contents: read jobs: + check-generated-code: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Rerun all code generators we have + run: python3 ci/cached-builds/gen_gha_matrix_jobs.py + + - name: Check there aren't any modified files present + run: | + if [[ $(git ls-files . -d -m -o --exclude-standard --full-name -v | tee modified.log | wc -l) -gt 0 ]]; then + echo "There are changed files" + exit 1 + fi + + - name: Print modified files + if: ${{ failure() }} + run: | + cat modified.log + git diff + code-static-analysis: runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index 314393142..83e9fcb50 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ IMAGE_REGISTRY ?= quay.io/opendatahub/workbench-images RELEASE ?= 2024a +# additional user-specified caching parameters for $(CONTAINER_ENGINE) build +CONTAINER_BUILD_CACHE_ARGS ?= --no-cache # OS dependant: Generate date, select appropriate cmd to locate container engine ifeq ($(OS), Windows_NT) @@ -41,7 +43,7 @@ define build_image $(eval BUILD_ARGS := --build-arg BASE_IMAGE=$(BASE_IMAGE_NAME)), $(eval BUILD_ARGS :=) ) - $(CONTAINER_ENGINE) build --no-cache -t $(IMAGE_NAME) $(BUILD_ARGS) $(2) + $(CONTAINER_ENGINE) build $(CONTAINER_BUILD_CACHE_ARGS) -t $(IMAGE_NAME) $(BUILD_ARGS) $(2) endef # Push function for the notebok image: diff --git a/ci/cached-builds/containers.conf b/ci/cached-builds/containers.conf new file mode 100644 index 000000000..6f9a8c43e --- /dev/null +++ b/ci/cached-builds/containers.conf @@ -0,0 +1,22 @@ +# https://github.com/containers/common/blob/main/docs/containers.conf.5.md + +[containers] + +[engine] +# needed for reliability +retry=100 +# supposedly these images are faster to pull +compression_format="zstd:chunked" +compression_level=6 +# defaults to /var/tmp, which is small +image_copy_tmp_dir="storage" + +[machine] + +[network] +# workaround for missing pasta binary in linuxbrew +default_rootless_network_cmd="slirp4netns" + +[secrets] + +[configmaps] diff --git a/ci/cached-builds/dev_null_container_registry.go b/ci/cached-builds/dev_null_container_registry.go new file mode 100644 index 000000000..bb3049fc0 --- /dev/null +++ b/ci/cached-builds/dev_null_container_registry.go @@ -0,0 +1,14 @@ +package main + +import ( + "log" + "net/http" +) + +func main() { + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + log.Printf("%s %v", r.Method, r.URL) + }) + + log.Fatal(http.ListenAndServe(":5000", nil)) +} diff --git a/ci/cached-builds/gen_gha_matrix_jobs.py b/ci/cached-builds/gen_gha_matrix_jobs.py new file mode 100644 index 000000000..867577932 --- /dev/null +++ b/ci/cached-builds/gen_gha_matrix_jobs.py @@ -0,0 +1,161 @@ +import itertools +import json +import os +import pathlib +import re +import string +from typing import Iterable + +"""Trivial Makefile parser that extracts target dependencies so that we can build each Dockerfile image target in its +own GitHub Actions job and handle dependencies between them. + +The parsing is not able to handle general Makefiles, it only works with the Makefile in this project. +Use https://pypi.org/project/py-make/ or https://github.com/JetBrains/intellij-plugins/tree/master/makefile/grammars if you look for general parser.""" + +project_dir = pathlib.Path(__file__).parent.parent.parent.absolute() + + +def read_makefile_lines(lines: Iterable[str]) -> list[str]: + """Processes line continuations lines and line comments + Note that this does not handle escaped backslash and escaped hash, or hash inside literals, ...""" + output = [] + current = "" + for line in lines: + # remove comment + if (i := line.find("#")) != -1: + line = line[:i] + + # line continuation + if line.endswith("\\\n"): + current += line[:-2] + else: + current += line[:-1] + output.append(current) + current = "" + if current: + output.append(current) + return output + + +def extract_target_dependencies(lines: Iterable[str]) -> dict[str, list[str]]: + tree = {} + for line in lines: + # not a target + if line.startswith("\t"): + continue + # .PHONY targets and such + if line.startswith("."): + continue + + r = re.compile(r""" + ^ # match from beginning + ([-A-Za-z0-9.]+)\s*: # target name + (?:\s* # any number of spaces between dependent targets + ([-A-Za-z0-9.]+) # dependent target name(s) + )* # ... + \s*$ # any whitespace at the end of the line + """, re.VERBOSE) + if m := re.match(r, line): + target, *deps = m.groups() + if deps == [None]: + deps = [] + tree[target] = deps + return tree + + +def write_github_workflow_file(tree: dict[str, list[str]], path: pathlib.Path) -> None: + jobs = {} + + # IDs may only contain alphanumeric characters, '_', and '-'. IDs must start with a letter or '_' and must be less than 100 characters. + allowed_github_chars = string.ascii_letters + string.digits + "_-" + + for task, deps in tree.items(): + # in level 0, we only want base images, not other utility tasks + if not deps: + if not task.startswith("base-"): + continue + + # we won't build rhel-based images because they need subscription + if "rhel" in task: + continue + + task_name = re.sub(r"[^-_0-9A-Za-z]", "_", task) + deps_names = [re.sub(r"[^-_0-9A-Za-z]", "_", dep) for dep in deps] + jobs[task_name] = { + "needs": deps_names, + "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml", + "with": { + "target": task, + "github": "${{ toJSON(github) }}", + }, + "secrets": "inherit", + } + + workflow = { + "name": "Build Notebooks", + # https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token + "permissions": { + "packages": "write", + }, + "on": { + "push": {}, + "workflow_dispatch": {}, + }, + "jobs": jobs, + } + + with open(path, "wt") as f: + print("---", file=f) + print("# This file is autogenerated by", pathlib.Path(__file__).relative_to(project_dir), file=f) + # every json file is a valid yaml file + json.dump(workflow, f, sort_keys=False, indent=4) + print(file=f) + + +def flatten(list_of_lists): + return list(itertools.chain.from_iterable(list_of_lists)) + +def compute_leafs_in_dependency_tree(tree: dict[str, list[str]]) -> list[str]: + key_set = set(tree.keys()) + value_set = set(flatten(tree.values())) + return [key for key in key_set if key not in value_set] + +def print_github_actions_pr_matrix(tree: dict[str, list[str]], leafs: list[str]) -> list[str]: + """Outputs GitHub matrix definition Json + """ + targets = [] + for leaf in leafs: + # in level 0, we only want base images, not other utility tasks + if not tree[leaf] and not leaf.startswith("base-"): + continue + + # we won't build rhel-based images because they need a subscription + if "rhel" in leaf: + continue + + targets.append(leaf) + + matrix = {"target": targets} + return [f"matrix={json.dumps(matrix, separators=(',', ':'))}"] + + +def main() -> None: + # https://www.gnu.org/software/make/manual/make.html#Reading-Makefiles + with open("Makefile", "rt") as makefile: + lines = read_makefile_lines(makefile) + tree = extract_target_dependencies(lines) + + write_github_workflow_file(tree, project_dir / ".github" / "workflows" / "build-notebooks.yaml") + + leafs = compute_leafs_in_dependency_tree(tree) + output = print_github_actions_pr_matrix(tree, leafs) + + print("leafs", leafs) + print(*output, sep="\n") + with open(os.environ["GITHUB_OUTPUT"], "at") as f: + for line in output: + print(line, file=f) + + +if __name__ == '__main__': + main() diff --git a/ci/cached-builds/insecure_localhost_registry.conf b/ci/cached-builds/insecure_localhost_registry.conf new file mode 100644 index 000000000..cddc459e8 --- /dev/null +++ b/ci/cached-builds/insecure_localhost_registry.conf @@ -0,0 +1,3 @@ +[[registry]] +location = "localhost:5000" +insecure = true diff --git a/ci/cached-builds/storage.conf b/ci/cached-builds/storage.conf new file mode 100644 index 000000000..24a181ec0 --- /dev/null +++ b/ci/cached-builds/storage.conf @@ -0,0 +1,11 @@ +# https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md + +[storage] +driver="overlay" +rootless_storage_path="/mnt/containers" + +[storage.options] +# https://www.redhat.com/sysadmin/faster-container-image-pulls +pull_options = {enable_partial_images = "true", use_hard_links = "true", ostree_repos=""} + +[storage.options.overlay]