diff --git a/.github/workflows/build-notebooks-TEMPLATE.yaml b/.github/workflows/build-notebooks-TEMPLATE.yaml index e826d79b5..e7b804d07 100644 --- a/.github/workflows/build-notebooks-TEMPLATE.yaml +++ b/.github/workflows/build-notebooks-TEMPLATE.yaml @@ -76,11 +76,15 @@ jobs: # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request - name: "pull_request: make ${{ inputs.target }}" - run: "make ${{ inputs.target }}" + run: | + # start a black hole container registry as make target always does a push + go run ci/dev_null_container_registry.go & + # build and push the image + make ${{ inputs.target }} if: "${{ fromJson(inputs.github).event_name == 'pull_request' }}" env: - IMAGE_TAG: "${{ github.head_ref }}_${{ github.sha }}" - IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images/pr${{ fromJson(inputs.github).event.number }}" + IMAGE_TAG: "${{ github.sha }}" + IMAGE_REGISTRY: "localhost:5000/workbench-images" CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }}" - run: df -h diff --git a/.github/workflows/build-notebooks-pr.yaml b/.github/workflows/build-notebooks-pr.yaml new file mode 100644 index 000000000..e1380370a --- /dev/null +++ b/.github/workflows/build-notebooks-pr.yaml @@ -0,0 +1,28 @@ +--- +"name": "Build Notebooks", +"permissions": + "packages": "read" +"on": + "pull_request": + +jobs: + gen: + name: Generate job matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.gen.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - run: python3 gen_gha_matrix_jobs.py + id: gen + + # base images + build: + needs: [ "gen" ] + strategy: + fail-fast: false + matrix: "${{ fromJson(needs.gen.outputs.matrix) }}" + uses: ./.github/workflows/build-notebooks-TEMPLATE.yaml + with: + target: "${{ matrix.target }}" + secrets: inherit diff --git a/.github/workflows/build-notebooks.yaml b/.github/workflows/build-notebooks.yaml index 47c211e12..a52f3de89 100644 --- a/.github/workflows/build-notebooks.yaml +++ b/.github/workflows/build-notebooks.yaml @@ -7,7 +7,6 @@ }, "on": { "push": {}, - "pull_request": {}, "workflow_dispatch": {} }, "jobs": { diff --git a/ci/dev_null_container_registry.go b/ci/dev_null_container_registry.go new file mode 100644 index 000000000..bb3049fc0 --- /dev/null +++ b/ci/dev_null_container_registry.go @@ -0,0 +1,14 @@ +package main + +import ( + "log" + "net/http" +) + +func main() { + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + log.Printf("%s %v", r.Method, r.URL) + }) + + log.Fatal(http.ListenAndServe(":5000", nil)) +} diff --git a/ci/gen_gha_matrix_jobs.py b/ci/gen_gha_matrix_jobs.py index 872d062c2..8d766bb18 100644 --- a/ci/gen_gha_matrix_jobs.py +++ b/ci/gen_gha_matrix_jobs.py @@ -1,4 +1,5 @@ import json +import os import pathlib import re import string @@ -61,25 +62,6 @@ def extract_target_dependencies(lines: Iterable[str]) -> dict[str, list[str]]: return tree -def print_github_actions_matrix(levels: dict[str, int]) -> list[str]: - """Outputs GitHub matrix definition Json as per - """ - lines = [] - for level in set(levels.values()): - targets: list[str] = list(l for l, v in levels.items() if v == level) - - # in level 0, we only want base images, not other utility tasks - if level == 0: - targets = [t for t in targets if t.startswith("base-")] - - # we won't build rhel-based images because they need subscription - targets = [t for t in targets if "rhel" not in t] - - matrix = {"target": targets} - lines.append(f"level{level}={json.dumps(matrix, separators=(",", ":"))}") - return lines - - def write_github_workflow_file(tree: dict[str, list[str]], path: pathlib.Path) -> None: jobs = {} @@ -116,7 +98,6 @@ def write_github_workflow_file(tree: dict[str, list[str]], path: pathlib.Path) - }, "on": { "push": {}, - "pull_request": {}, "workflow_dispatch": {}, }, "jobs": jobs, @@ -129,6 +110,29 @@ def write_github_workflow_file(tree: dict[str, list[str]], path: pathlib.Path) - json.dump(workflow, f, sort_keys=False, indent=4) print(file=f) +def compute_leafs_in_dependency_tree(tree: dict[str, list[str]]) -> list[str]: + key_set = set(tree.keys()) + value_set = set(*tree.values()) + return [key for key in key_set if key not in value_set] + +def print_github_actions_pr_matrix(tree: dict[str, list[str]], leafs: list[str]) -> str: + """Outputs GitHub matrix definition Json + """ + targets = [] + for leaf in leafs: + # in level 0, we only want base images, not other utility tasks + if not tree[leaf] and leaf.startswith("base-"): + continue + + # we won't build rhel-based images because they need subscription + if "rhel" in leaf: + continue + + targets.append(leaf) + + matrix = {"target": targets} + return f"matrix={json.dumps(matrix, separators=(",", ":"))}" + def main() -> None: # https://www.gnu.org/software/make/manual/make.html#Reading-Makefiles @@ -138,6 +142,16 @@ def main() -> None: write_github_workflow_file(tree, project_dir / ".github" / "workflows" / "build-notebooks.yaml") + leafs = compute_leafs_in_dependency_tree(tree) + output = print_github_actions_pr_matrix(leafs) + + write_github_workflow_file(tree) + + print(*output, sep="\n") + with open(os.environ["GITHUB_OUTPUT"], "at") as f: + for line in output: + print(line, file=f) + if __name__ == '__main__': main() diff --git a/ci/insecure-localhost-registry.conf b/ci/insecure-localhost-registry.conf new file mode 100644 index 000000000..cddc459e8 --- /dev/null +++ b/ci/insecure-localhost-registry.conf @@ -0,0 +1,3 @@ +[[registry]] +location = "localhost:5000" +insecure = true