Generate Konflux configurations

Integrate KonfluxGen with prowgen so that after running prowgen we can create PRs for all midstream repositories to add/update/remove Konflux components and applications (to be added pipelines and ImageRepository). Signed-off-by: Pierangelo Di Pilato <[email protected]>
openshift-knative · Jun 14, 2024 · e37dbe1 · e37dbe1
1 parent 211eb9b
commit e37dbe1
Show file tree

Hide file tree

Showing 18 changed files with 850 additions and 79 deletions.
diff --git a/.github/workflows/release-generate-ci-template.yaml b/.github/workflows/release-generate-ci-template.yaml
@@ -0,0 +1,80 @@
+name: Generate CI config (template)
+
+on:
+  pull_request:
+    branches:
+      - '**'
+  push:
+    branches:
+      - 'main'
+  schedule:
+    - cron: "0 6 * * *" # Daily at 06:00.
+  workflow_dispatch: # Manual workflow trigger
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  generate-ci:
+    name: generate-ci
+    runs-on: ubuntu-latest
+    env:
+      GOPATH: ${{ github.workspace }}
+    steps:
+      - name: Install prerequisites
+        env:
+          YQ_VERSION: 3.4.0
+        run: |
+          sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq
+          sudo chmod +x /usr/bin/yq
+          sudo mv /usr/bin/yq /usr/local/bin/yq
+
+      - name: Checkout openshift-knative/hack
+        uses: actions/checkout@v3
+        with:
+          path: ./src/github.com/openshift-knative/hack
+
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: ./src/github.com/openshift-knative/hack/go.mod
+
+      - name: Install YAML2JSON
+        run: go install github.com/bronze1man/yaml2json@latest
+
+      - name: Convert configurations to JSON
+        working-directory: ./src/github.com/openshift-knative/hack
+        run: find config/*.yaml | xargs -I{} sh -c "yaml2json < {} > {}.json"
+
+      - name: Checkout openshift/release
+        uses: actions/checkout@v3
+        with:
+          branch: 'master'
+          repository: 'openshift/release'
+          path: ./src/github.com/openshift-knative/hack/openshift/release
+
+      - name: Configure Git user
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "OpenShift Serverless"
+
+      - name: Generate CI
+        working-directory: ./src/github.com/openshift-knative/hack
+        # Use master, see https://github.com/peter-evans/create-pull-request/issues/2108
+        run: make generate-ci-no-clean ARGS=--branch=master
+
+      - name: Create Pull Request
+        if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_name == 'main'
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.SERVERLESS_QE_ROBOT }}
+          path: ./src/github.com/openshift-knative/hack/openshift/release
+          base: master
+          branch: sync-serverless-ci
+          title: "Sync Serverless CI"
+          commit-message: "Sync Serverless CI"
+          push-to-fork: serverless-qe/release
+          delete-branch: true
+          body: |
+            Sync Serverless CI using openshift-knative/hack.
diff --git a/.github/workflows/release-generate-ci.yaml b/.github/workflows/release-generate-ci.yaml
@@ -1,70 +1,82 @@
----
 name: Generate CI config
-
 on:
-  pull_request:
-    branches:
-      - '**'
-  push:
-    branches:
-      - 'main'
-  schedule:
-    - cron: "0 6 * * *" # Daily at 06:00.
-  workflow_dispatch: # Manual workflow trigger
-
+    pull_request:
+        branches:
+            - '**'
+    push:
+        branches:
+            - 'main'
+    schedule:
+        - cron: "0 6 * * *" # Daily at 06:00.
+    workflow_dispatch: # Manual workflow trigger
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
 jobs:
-  generate-ci:
-    name: generate-ci
-    runs-on: ubuntu-latest
-    env:
-      GOPATH: ${{ github.workspace }}
-    steps:
-      - name: Install prerequisites
+    generate-ci:
+        name: generate-ci
+        runs-on: ubuntu-latest
         env:
-          YQ_VERSION: 3.4.0
-        run: |
-          sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq
-          sudo chmod +x /usr/bin/yq
-          sudo mv /usr/bin/yq /usr/local/bin/yq
-
-      - name: Checkout openshift-knative/hack
-        uses: actions/checkout@v3
-        with:
-          path: ./src/github.com/openshift-knative/hack
-
-      - name: Setup Golang
-        uses: actions/setup-go@v5
-        with:
-          go-version-file: ./src/github.com/openshift-knative/hack/go.mod
-
-      - name: Checkout openshift/release
-        uses: actions/checkout@v3
-        with:
-          branch: 'master'
-          repository: 'openshift/release'
-          path: ./src/github.com/openshift-knative/hack/openshift/release
-
-      - name: Configure Git user
-        run: |
-          git config --global user.email "[email protected]"
-          git config --global user.name "OpenShift Serverless"
-
-      - name: Generate CI
-        working-directory: ./src/github.com/openshift-knative/hack
-        # Use master, see https://github.com/peter-evans/create-pull-request/issues/2108
-        run: make generate-ci-no-clean ARGS=--branch=master
-
-      - name: Create Pull Request
-        if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_name == 'main'
-        uses: peter-evans/create-pull-request@v5
-        with:
-          token: ${{ secrets.SERVERLESS_QE_ROBOT }}
-          path: ./src/github.com/openshift-knative/hack/openshift/release
-          base: master
-          branch: sync-serverless-ci
-          title: "Sync Serverless CI"
-          commit-message: "Sync Serverless CI"
-          push-to-fork: serverless-qe/release
-          delete-branch: true
-          body: |
-            Sync Serverless CI using openshift-knative/hack.
+            GOPATH: ${{ github.workspace }}
+        steps:
+            - name: Install prerequisites
+              env:
+                YQ_VERSION: 3.4.0
+              run: |
+                sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq
+                sudo chmod +x /usr/bin/yq
+                sudo mv /usr/bin/yq /usr/local/bin/yq
+            - name: Checkout openshift-knative/hack
+              uses: actions/checkout@v3
+              with:
+                path: ./src/github.com/openshift-knative/hack
+            - name: Setup Golang
+              uses: actions/setup-go@v5
+              with:
+                go-version-file: ./src/github.com/openshift-knative/hack/go.mod
+            - name: Install YAML2JSON
+              run: go install github.com/bronze1man/yaml2json@latest
+            - name: Convert configurations to JSON
+              working-directory: ./src/github.com/openshift-knative/hack
+              run: find config/*.yaml | xargs -I{} sh -c "yaml2json < {} > {}.json"
+            - name: Checkout openshift/release
+              uses: actions/checkout@v3
+              with:
+                branch: 'master'
+                repository: 'openshift/release'
+                path: ./src/github.com/openshift-knative/hack/openshift/release
+            - name: Configure Git user
+              run: |
+                git config --global user.email "[email protected]"
+                git config --global user.name "OpenShift Serverless"
+            - name: Generate CI
+              working-directory: ./src/github.com/openshift-knative/hack
+              # Use master, see https://github.com/peter-evans/create-pull-request/issues/2108
+              run: make generate-ci-no-clean ARGS=--branch=master
+            - name: Create Pull Request
+              if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_name == 'main'
+              uses: peter-evans/create-pull-request@v5
+              with:
+                token: ${{ secrets.SERVERLESS_QE_ROBOT }}
+                path: ./src/github.com/openshift-knative/hack/openshift/release
+                base: master
+                branch: sync-serverless-ci
+                title: "Sync Serverless CI"
+                commit-message: "Sync Serverless CI"
+                push-to-fork: serverless-qe/release
+                delete-branch: true
+                body: |
+                    Sync Serverless CI using openshift-knative/hack.
+            - if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_name == 'main'
+              name: '[eventing - release-next] Create Konflux PR'
+              uses: peter-evans/create-pull-request@v5
+              with:
+                base: main
+                body: '[main] Sync Konflux configurations'
+                branch: sync-konflux/release-next
+                commit-message: '[main] Sync Konflux configurations'
+                delete-branch: true
+                path: /src/github.com/openshift-knative/hack/openshift-knative/eventing
+                push-to-fork: serverless-qe/eventing
+                title: '[main] Sync Konflux configurations'
+                token: ${{ secrets.SERVERLESS_QE_ROBOT }}
diff --git a/.gitignore b/.gitignore
@@ -16,6 +16,7 @@
 
 /openshift
 /openshift-knative
+/konflux-gen/out
 
 # MacOS
 .DS_Store
diff --git a/Makefile b/Makefile
@@ -30,13 +30,25 @@ discover-branches:
 	go run github.com/openshift-knative/hack/cmd/discover $(ARGS)
 .PHONY: discover-branches
 
+generate-action:
+	go run github.com/openshift-knative/hack/cmd/update-konflux-gen-action
+.PHONY: generate-action
+
 unit-tests:
 	go test ./pkg/...
 
 	rm -rf openshift/project/testoutput
+	rm -rf openshift/project/.github
+
+	mkdir -p openshift
+	go run ./cmd/update-konflux-gen-action --input ".github/workflows/release-generate-ci-template.yaml" --config "config/" --output "openshift/release-generate-ci.yaml"
+	# If the following fails, please run 'make generate-action'
+	diff -r "openshift/release-generate-ci.yaml" ".github/workflows/release-generate-ci.yaml"
+
 	go run ./cmd/generate/ --generators dockerfile \
 		--project-file pkg/project/testdata/project.yaml \
 		--excludes ".*vendor.*" \
+		--excludes ".*konflux-gen.*" \
 		--excludes "openshift.*" \
 		--images-from "hack" \
 		--images-from-url-format "https://raw.githubusercontent.com/openshift-knative/%s/%s/pkg/project/testdata/additional-images.yaml" \

diff --git a/cmd/konflux-gen/README.md b/cmd/konflux-gen/README.md
@@ -0,0 +1,26 @@
+# Konflux Gen
+
+A CLI to get started
+using [Konflux](https://redhat-appstudio.github.io/docs.appstudio.io/Documentation/main/).
+
+## Usage
+
+### Generate applications and components
+
+```shell
+# Clone openshift/release repository in openshift-release
+git clone [email protected]:openshift/release.git openshift-release
+konflux-gen --openshift-release-path openshift-release --includes "ci-operator/config/<org>/.*.yaml" --output "$(pwd)/<org>"
+```
+
+Example command:
+
+```shell
+go run ./cmd/konflux-gen/main.go --openshift-release-path openshift-release \
+  --application-name "serverless-operator release-1.32" \
+  --includes "ci-operator/config/openshift-knative/.*v1.11.*.yaml" \
+  --includes "ci-operator/config/openshift-knative/serverless-operator/.*1.32.*.yaml" \
+  --exclude-images ".*source.*" \
+  --exclude-images ".*test.*" \
+  --output konflux-gen/out
+```
diff --git a/cmd/konflux-gen/main.go b/cmd/konflux-gen/main.go
@@ -0,0 +1,47 @@
+package main
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/spf13/pflag"
+
+	"github.com/openshift-knative/hack/pkg/konfluxgen"
+)
+
+const (
+	openShiftReleasePathFlag = "openshift-release-path"
+	applicationNameFlag      = "application-name"
+	includesFlag             = "includes"
+	excludesFlag             = "excludes"
+	excludeImagesFlag        = "exclude-images"
+	outputFlag               = "output"
+)
+
+func main() {
+	if err := run(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func run() error {
+
+	cfg := konfluxgen.Config{}
+
+	pflag.StringVar(&cfg.OpenShiftReleasePath, openShiftReleasePathFlag, "", "openshift/release repository path")
+	pflag.StringVar(&cfg.ApplicationName, applicationNameFlag, "", "Konflux application name")
+	pflag.StringVar(&cfg.ResourcesOutputPath, outputFlag, "", "output path")
+	pflag.StringArrayVar(&cfg.Includes, includesFlag, nil, "Regex to select CI config files to include")
+	pflag.StringArrayVar(&cfg.Excludes, excludesFlag, nil, "Regex to select CI config files to exclude")
+	pflag.StringArrayVar(&cfg.ExcludesImages, excludeImagesFlag, nil, "Regex to select CI config images to exclude")
+	pflag.Parse()
+
+	if cfg.OpenShiftReleasePath == "" {
+		return fmt.Errorf("expected %q flag to be non empty", openShiftReleasePathFlag)
+	}
+	if len(cfg.Includes) == 0 {
+		return fmt.Errorf("expected %q flag to be non empty", includesFlag)
+	}
+
+	return konfluxgen.Generate(cfg)
+}
diff --git a/cmd/sobranch/sobranch.go b/cmd/sobranch/sobranch.go
@@ -3,23 +3,14 @@ package main
 import (
 	"flag"
 	"fmt"
+
 	"github.com/openshift-knative/hack/pkg/sobranch"
-	"strings"
 )
 
 func main() {
 	upstreamVersion := flag.String("upstream-version", "", "Upstream version")
 	flag.Parse()
 
-	*upstreamVersion = strings.Replace(*upstreamVersion, "release-v", "", 1)
-	*upstreamVersion = strings.Replace(*upstreamVersion, "release-", "", 1)
-	*upstreamVersion = strings.Replace(*upstreamVersion, "v", "", 1)
-
-	dotParts := strings.SplitN(*upstreamVersion, ".", 3)
-	if len(dotParts) == 2 {
-		*upstreamVersion = *upstreamVersion + ".0"
-	}
-
 	soBranch := sobranch.FromUpstreamVersion(*upstreamVersion)
 
 	fmt.Printf(soBranch)

diff --git a/cmd/update-konflux-gen-action/main.go b/cmd/update-konflux-gen-action/main.go
@@ -0,0 +1,26 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"path/filepath"
+
+	"github.com/openshift-knative/hack/pkg/action"
+)
+
+func main() {
+
+	inputConfig := flag.String("config", filepath.Join("config"), "Specify repositories config")
+	inputAction := flag.String("input", filepath.Join(".github", "workflows", "release-generate-ci-template.yaml"), "Input action (template)")
+	outputAction := flag.String("output", filepath.Join(".github", "workflows", "release-generate-ci.yaml"), "Output action")
+	flag.Parse()
+
+	err := action.UpdateAction(action.Config{
+		InputAction:     *inputAction,
+		InputConfigPath: *inputConfig,
+		OutputAction:    *outputAction,
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+}