diff --git a/.github/workflows/release-generate-ci-template.yaml b/.github/workflows/release-generate-ci-template.yaml new file mode 100644 index 00000000..88c72b73 --- /dev/null +++ b/.github/workflows/release-generate-ci-template.yaml @@ -0,0 +1,80 @@ +name: Generate CI config (template) + +on: + pull_request: + branches: + - '**' + push: + branches: + - 'main' + schedule: + - cron: "0 6 * * *" # Daily at 06:00. + workflow_dispatch: # Manual workflow trigger + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + generate-ci: + name: generate-ci + runs-on: ubuntu-latest + env: + GOPATH: ${{ github.workspace }} + steps: + - name: Install prerequisites + env: + YQ_VERSION: 3.4.0 + run: | + sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq + sudo mv /usr/bin/yq /usr/local/bin/yq + + - name: Checkout openshift-knative/hack + uses: actions/checkout@v3 + with: + path: ./src/github.com/openshift-knative/hack + + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version-file: ./src/github.com/openshift-knative/hack/go.mod + + - name: Install YAML2JSON + run: go install github.com/bronze1man/yaml2json@latest + + - name: Convert configurations to JSON + working-directory: ./src/github.com/openshift-knative/hack + run: find config/*.yaml | xargs -I{} sh -c "yaml2json < {} > {}.json" + + - name: Checkout openshift/release + uses: actions/checkout@v3 + with: + branch: 'master' + repository: 'openshift/release' + path: ./src/github.com/openshift-knative/hack/openshift/release + + - name: Configure Git user + run: | + git config --global user.email "serverless-support@redhat.com" + git config --global user.name "OpenShift Serverless" + + - name: Generate CI + working-directory: ./src/github.com/openshift-knative/hack + # Use master, see https://github.com/peter-evans/create-pull-request/issues/2108 + run: make generate-ci-no-clean ARGS=--branch=master + + - name: Create Pull Request + if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_name == 'main' + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.SERVERLESS_QE_ROBOT }} + path: ./src/github.com/openshift-knative/hack/openshift/release + base: master + branch: sync-serverless-ci + title: "Sync Serverless CI" + commit-message: "Sync Serverless CI" + push-to-fork: serverless-qe/release + delete-branch: true + body: | + Sync Serverless CI using openshift-knative/hack. diff --git a/.github/workflows/release-generate-ci.yaml b/.github/workflows/release-generate-ci.yaml index f74f584a..dfec8ee5 100644 --- a/.github/workflows/release-generate-ci.yaml +++ b/.github/workflows/release-generate-ci.yaml @@ -1,70 +1,82 @@ ---- name: Generate CI config - on: - pull_request: - branches: - - '**' - push: - branches: - - 'main' - schedule: - - cron: "0 6 * * *" # Daily at 06:00. - workflow_dispatch: # Manual workflow trigger - + pull_request: + branches: + - '**' + push: + branches: + - 'main' + schedule: + - cron: "0 6 * * *" # Daily at 06:00. + workflow_dispatch: # Manual workflow trigger +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - generate-ci: - name: generate-ci - runs-on: ubuntu-latest - env: - GOPATH: ${{ github.workspace }} - steps: - - name: Install prerequisites + generate-ci: + name: generate-ci + runs-on: ubuntu-latest env: - YQ_VERSION: 3.4.0 - run: | - sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - sudo mv /usr/bin/yq /usr/local/bin/yq - - - name: Checkout openshift-knative/hack - uses: actions/checkout@v3 - with: - path: ./src/github.com/openshift-knative/hack - - - name: Setup Golang - uses: actions/setup-go@v5 - with: - go-version-file: ./src/github.com/openshift-knative/hack/go.mod - - - name: Checkout openshift/release - uses: actions/checkout@v3 - with: - branch: 'master' - repository: 'openshift/release' - path: ./src/github.com/openshift-knative/hack/openshift/release - - - name: Configure Git user - run: | - git config --global user.email "serverless-support@redhat.com" - git config --global user.name "OpenShift Serverless" - - - name: Generate CI - working-directory: ./src/github.com/openshift-knative/hack - # Use master, see https://github.com/peter-evans/create-pull-request/issues/2108 - run: make generate-ci-no-clean ARGS=--branch=master - - - name: Create Pull Request - if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_name == 'main' - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.SERVERLESS_QE_ROBOT }} - path: ./src/github.com/openshift-knative/hack/openshift/release - base: master - branch: sync-serverless-ci - title: "Sync Serverless CI" - commit-message: "Sync Serverless CI" - push-to-fork: serverless-qe/release - delete-branch: true - body: | - Sync Serverless CI using openshift-knative/hack. + GOPATH: ${{ github.workspace }} + steps: + - name: Install prerequisites + env: + YQ_VERSION: 3.4.0 + run: | + sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq + sudo mv /usr/bin/yq /usr/local/bin/yq + - name: Checkout openshift-knative/hack + uses: actions/checkout@v3 + with: + path: ./src/github.com/openshift-knative/hack + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version-file: ./src/github.com/openshift-knative/hack/go.mod + - name: Install YAML2JSON + run: go install github.com/bronze1man/yaml2json@latest + - name: Convert configurations to JSON + working-directory: ./src/github.com/openshift-knative/hack + run: find config/*.yaml | xargs -I{} sh -c "yaml2json < {} > {}.json" + - name: Checkout openshift/release + uses: actions/checkout@v3 + with: + branch: 'master' + repository: 'openshift/release' + path: ./src/github.com/openshift-knative/hack/openshift/release + - name: Configure Git user + run: | + git config --global user.email "serverless-support@redhat.com" + git config --global user.name "OpenShift Serverless" + - name: Generate CI + working-directory: ./src/github.com/openshift-knative/hack + # Use master, see https://github.com/peter-evans/create-pull-request/issues/2108 + run: make generate-ci-no-clean ARGS=--branch=master + - name: Create Pull Request + if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_name == 'main' + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.SERVERLESS_QE_ROBOT }} + path: ./src/github.com/openshift-knative/hack/openshift/release + base: master + branch: sync-serverless-ci + title: "Sync Serverless CI" + commit-message: "Sync Serverless CI" + push-to-fork: serverless-qe/release + delete-branch: true + body: | + Sync Serverless CI using openshift-knative/hack. + - if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_name == 'main' + name: '[eventing - release-next] Create Konflux PR' + uses: peter-evans/create-pull-request@v5 + with: + base: main + body: '[main] Sync Konflux configurations' + branch: sync-konflux/release-next + commit-message: '[main] Sync Konflux configurations' + delete-branch: true + path: /src/github.com/openshift-knative/hack/openshift-knative/eventing + push-to-fork: serverless-qe/eventing + title: '[main] Sync Konflux configurations' + token: ${{ secrets.SERVERLESS_QE_ROBOT }} diff --git a/.gitignore b/.gitignore index 31c49ab9..73ec65b6 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ /openshift /openshift-knative +/konflux-gen/out # MacOS .DS_Store diff --git a/Makefile b/Makefile index f8a4a225..0a4884b3 100644 --- a/Makefile +++ b/Makefile @@ -30,13 +30,25 @@ discover-branches: go run github.com/openshift-knative/hack/cmd/discover $(ARGS) .PHONY: discover-branches +generate-action: + go run github.com/openshift-knative/hack/cmd/update-konflux-gen-action +.PHONY: generate-action + unit-tests: go test ./pkg/... rm -rf openshift/project/testoutput + rm -rf openshift/project/.github + + mkdir -p openshift + go run ./cmd/update-konflux-gen-action --input ".github/workflows/release-generate-ci-template.yaml" --config "config/" --output "openshift/release-generate-ci.yaml" + # If the following fails, please run 'make generate-action' + diff -r "openshift/release-generate-ci.yaml" ".github/workflows/release-generate-ci.yaml" + go run ./cmd/generate/ --generators dockerfile \ --project-file pkg/project/testdata/project.yaml \ --excludes ".*vendor.*" \ + --excludes ".*konflux-gen.*" \ --excludes "openshift.*" \ --images-from "hack" \ --images-from-url-format "https://raw.githubusercontent.com/openshift-knative/%s/%s/pkg/project/testdata/additional-images.yaml" \ diff --git a/cmd/konflux-gen/README.md b/cmd/konflux-gen/README.md new file mode 100644 index 00000000..acedf216 --- /dev/null +++ b/cmd/konflux-gen/README.md @@ -0,0 +1,26 @@ +# Konflux Gen + +A CLI to get started +using [Konflux](https://redhat-appstudio.github.io/docs.appstudio.io/Documentation/main/). + +## Usage + +### Generate applications and components + +```shell +# Clone openshift/release repository in openshift-release +git clone git@github.com:openshift/release.git openshift-release +konflux-gen --openshift-release-path openshift-release --includes "ci-operator/config//.*.yaml" --output "$(pwd)/" +``` + +Example command: + +```shell +go run ./cmd/konflux-gen/main.go --openshift-release-path openshift-release \ + --application-name "serverless-operator release-1.32" \ + --includes "ci-operator/config/openshift-knative/.*v1.11.*.yaml" \ + --includes "ci-operator/config/openshift-knative/serverless-operator/.*1.32.*.yaml" \ + --exclude-images ".*source.*" \ + --exclude-images ".*test.*" \ + --output konflux-gen/out +``` \ No newline at end of file diff --git a/cmd/konflux-gen/main.go b/cmd/konflux-gen/main.go new file mode 100644 index 00000000..4bd30f48 --- /dev/null +++ b/cmd/konflux-gen/main.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "log" + + "github.com/spf13/pflag" + + "github.com/openshift-knative/hack/pkg/konfluxgen" +) + +const ( + openShiftReleasePathFlag = "openshift-release-path" + applicationNameFlag = "application-name" + includesFlag = "includes" + excludesFlag = "excludes" + excludeImagesFlag = "exclude-images" + outputFlag = "output" +) + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() error { + + cfg := konfluxgen.Config{} + + pflag.StringVar(&cfg.OpenShiftReleasePath, openShiftReleasePathFlag, "", "openshift/release repository path") + pflag.StringVar(&cfg.ApplicationName, applicationNameFlag, "", "Konflux application name") + pflag.StringVar(&cfg.ResourcesOutputPath, outputFlag, "", "output path") + pflag.StringArrayVar(&cfg.Includes, includesFlag, nil, "Regex to select CI config files to include") + pflag.StringArrayVar(&cfg.Excludes, excludesFlag, nil, "Regex to select CI config files to exclude") + pflag.StringArrayVar(&cfg.ExcludesImages, excludeImagesFlag, nil, "Regex to select CI config images to exclude") + pflag.Parse() + + if cfg.OpenShiftReleasePath == "" { + return fmt.Errorf("expected %q flag to be non empty", openShiftReleasePathFlag) + } + if len(cfg.Includes) == 0 { + return fmt.Errorf("expected %q flag to be non empty", includesFlag) + } + + return konfluxgen.Generate(cfg) +} diff --git a/cmd/sobranch/sobranch.go b/cmd/sobranch/sobranch.go index cf9d738e..0ca9d328 100644 --- a/cmd/sobranch/sobranch.go +++ b/cmd/sobranch/sobranch.go @@ -3,23 +3,14 @@ package main import ( "flag" "fmt" + "github.com/openshift-knative/hack/pkg/sobranch" - "strings" ) func main() { upstreamVersion := flag.String("upstream-version", "", "Upstream version") flag.Parse() - *upstreamVersion = strings.Replace(*upstreamVersion, "release-v", "", 1) - *upstreamVersion = strings.Replace(*upstreamVersion, "release-", "", 1) - *upstreamVersion = strings.Replace(*upstreamVersion, "v", "", 1) - - dotParts := strings.SplitN(*upstreamVersion, ".", 3) - if len(dotParts) == 2 { - *upstreamVersion = *upstreamVersion + ".0" - } - soBranch := sobranch.FromUpstreamVersion(*upstreamVersion) fmt.Printf(soBranch) diff --git a/cmd/update-konflux-gen-action/main.go b/cmd/update-konflux-gen-action/main.go new file mode 100644 index 00000000..127caac6 --- /dev/null +++ b/cmd/update-konflux-gen-action/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "flag" + "log" + "path/filepath" + + "github.com/openshift-knative/hack/pkg/action" +) + +func main() { + + inputConfig := flag.String("config", filepath.Join("config"), "Specify repositories config") + inputAction := flag.String("input", filepath.Join(".github", "workflows", "release-generate-ci-template.yaml"), "Input action (template)") + outputAction := flag.String("output", filepath.Join(".github", "workflows", "release-generate-ci.yaml"), "Output action") + flag.Parse() + + err := action.UpdateAction(action.Config{ + InputAction: *inputAction, + InputConfigPath: *inputConfig, + OutputAction: *outputAction, + }) + if err != nil { + log.Fatal(err) + } +} diff --git a/config/eventing.yaml b/config/eventing.yaml index aaff0c9f..6d69141e 100644 --- a/config/eventing.yaml +++ b/config/eventing.yaml @@ -1,6 +1,8 @@ config: branches: release-next: + konflux: + enabled: true openShiftVersions: - version: "4.15" - version: "4.12" diff --git a/pkg/action/update_action.go b/pkg/action/update_action.go new file mode 100644 index 00000000..4a62cceb --- /dev/null +++ b/pkg/action/update_action.go @@ -0,0 +1,141 @@ +package action + +import ( + "bytes" + "fmt" + "io/fs" + "os" + "path/filepath" + + "gopkg.in/yaml.v3" + + "github.com/openshift-knative/hack/pkg/prowgen" +) + +type Config struct { + InputAction string + InputConfigPath string + OutputAction string +} + +func UpdateAction(cfg Config) error { + var steps []interface{} + + y, err := os.ReadFile(cfg.InputAction) + if err != nil { + return err + } + var node yaml.Node + if err := yaml.NewDecoder(bytes.NewBuffer(y)).Decode(&node); err != nil { + return fmt.Errorf("failed to decode file into node: %w", err) + } + + if err := AddNestedField(&node, "Generate CI config", "name"); err != nil { + return fmt.Errorf("failed to add steps: %w", err) + } + + err = filepath.Walk(cfg.InputConfigPath, func(path string, info fs.FileInfo, err error) error { + if info.IsDir() { + return nil + } + + inConfig, err := prowgen.LoadConfig(path) + if err != nil { + return err + } + + for _, r := range inConfig.Repositories { + for branchName, b := range inConfig.Config.Branches { + if b.Konflux.Enabled { + + // Special case "release-next" + targetBranch := branchName + if branchName == "release-next" { + targetBranch = "main" + } + + commit := fmt.Sprintf("[%s] Sync Konflux configurations", targetBranch) + + steps = append(steps, map[string]interface{}{ + "name": fmt.Sprintf("[%s - %s] Create Konflux PR", r.Repo, branchName), + "if": "(github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_name == 'main'", + "uses": "peter-evans/create-pull-request@v5", + "with": map[string]interface{}{ + "token": "${{ secrets.SERVERLESS_QE_ROBOT }}", + "path": fmt.Sprintf("/src/github.com/openshift-knative/hack/%s", r.RepositoryDirectory()), + "base": targetBranch, + "branch": fmt.Sprintf("%s%s", prowgen.KonfluxBranchPrefix, branchName), + "title": commit, + "commit-message": commit, + "push-to-fork": fmt.Sprintf("serverless-qe/%s", r.Repo), + "delete-branch": true, + "body": commit, + }, + }) + } + } + } + + return nil + }) + if err != nil { + return fmt.Errorf("failed to walk filesystem path %q: %w", cfg.InputConfigPath, err) + } + + if err := AddNestedField(&node, steps, "jobs", "generate-ci", "steps"); err != nil { + return fmt.Errorf("failed to add steps: %w", err) + } + + buf := bytes.NewBuffer(nil) + if err := yaml.NewEncoder(buf).Encode(&node); err != nil { + return fmt.Errorf("failed to encode node into buf: %w", err) + } + + if err := os.WriteFile(cfg.OutputAction, buf.Bytes(), 0600); err != nil { + return fmt.Errorf("failed to write updates: %w", err) + } + + return nil +} + +func AddNestedField(node *yaml.Node, value interface{}, fields ...string) error { + + for i, n := range node.Content { + + if i > 0 && node.Content[i-1].Value == fields[0] { + + // Base case for scalar nodes + if len(fields) == 1 && n.Kind == yaml.ScalarNode { + n.SetString(fmt.Sprintf("%s", value)) + break + } + // base case for sequence node + if len(fields) == 1 && n.Kind == yaml.SequenceNode { + + if v, ok := value.([]interface{}); ok { + var s yaml.Node + + b, err := yaml.Marshal(v) + if err != nil { + return err + } + if err := yaml.NewDecoder(bytes.NewBuffer(b)).Decode(&s); err != nil { + return err + } + + n.Content = append(n.Content, s.Content[0].Content...) + } + break + } + + // Continue to the next level + return AddNestedField(n, value, fields[1:]...) + } + + if node.Kind == yaml.DocumentNode { + return AddNestedField(n, value, fields...) + } + } + + return nil +} diff --git a/pkg/discover/discover.go b/pkg/discover/discover.go index 20509c49..02b96e40 100644 --- a/pkg/discover/discover.go +++ b/pkg/discover/discover.go @@ -14,6 +14,7 @@ import ( gyaml "github.com/ghodss/yaml" + "github.com/openshift-knative/hack/pkg/action" "github.com/openshift-knative/hack/pkg/prowgen" ) @@ -23,6 +24,8 @@ func Main() { defer cancel() inputConfig := flag.String("config", filepath.Join("config"), "Specify repositories config") + inputAction := flag.String("input", filepath.Join(".github", "workflows", "release-generate-ci-template.yaml"), "Input action (template)") + outputAction := flag.String("output", filepath.Join(".github", "workflows", "release-generate-ci.yaml"), "Output action") flag.Parse() err := filepath.Walk(*inputConfig, func(path string, info fs.FileInfo, err error) error { @@ -39,6 +42,15 @@ func Main() { if err != nil { log.Fatalln("Failed to walk path", *inputConfig, err) } + + err = action.UpdateAction(action.Config{ + InputAction: *inputAction, + InputConfigPath: *inputConfig, + OutputAction: *outputAction, + }) + if err != nil { + log.Fatal(err) + } } func discover(ctx context.Context, path string) error { diff --git a/pkg/konfluxgen/application.template.yaml b/pkg/konfluxgen/application.template.yaml new file mode 100644 index 00000000..80921be1 --- /dev/null +++ b/pkg/konfluxgen/application.template.yaml @@ -0,0 +1,7 @@ +apiVersion: appstudio.redhat.com/v1alpha1 +kind: Application +metadata: + name: {{ truncate ( sanitize .ApplicationName ) }} +spec: + description: {{ .ApplicationName }} + displayName: {{ .ApplicationName }} diff --git a/pkg/konfluxgen/dockerfile-component.template.yaml b/pkg/konfluxgen/dockerfile-component.template.yaml new file mode 100644 index 00000000..e6b72845 --- /dev/null +++ b/pkg/konfluxgen/dockerfile-component.template.yaml @@ -0,0 +1,22 @@ +apiVersion: appstudio.redhat.com/v1alpha1 +kind: Component +metadata: + annotations: + image.redhat.com/generate: "true" + appstudio.openshift.io/pac-provision: request + name: {{ truncate ( sanitize .ProjectDirectoryImageBuildStepConfiguration.To ) }} +spec: + componentName: {{ .ProjectDirectoryImageBuildStepConfiguration.To }} + application: {{ sanitize .ApplicationName }} + {{ $nudgesN := len .Nudges }} {{ if ne $nudgesN 0 }} + build-nudges-ref: + {{ range $nudge := .Nudges }} + - "{{ $nudge }}" + {{ end }} + {{ end }} + source: + git: + url: https://github.com/{{ .ReleaseBuildConfiguration.Metadata.Org }}/{{ .ReleaseBuildConfiguration.Metadata.Repo }}.git + context: {{ .ProjectDirectoryImageBuildStepConfiguration.ProjectDirectoryImageBuildInputs.ContextDir }} + dockerfileUrl: {{ .ProjectDirectoryImageBuildStepConfiguration.ProjectDirectoryImageBuildInputs.DockerfilePath }} + revision: {{ .ReleaseBuildConfiguration.Metadata.Branch }} diff --git a/pkg/konfluxgen/konfluxgen.go b/pkg/konfluxgen/konfluxgen.go new file mode 100644 index 00000000..a8e7565b --- /dev/null +++ b/pkg/konfluxgen/konfluxgen.go @@ -0,0 +1,305 @@ +package konfluxgen + +import ( + "bytes" + "crypto/md5" + "embed" + "encoding/json" + "fmt" + "io/fs" + "log" + "os" + "path/filepath" + "regexp" + "strings" + "text/template" + + gyaml "github.com/ghodss/yaml" + cioperatorapi "github.com/openshift/ci-tools/pkg/api" +) + +//go:embed application.template.yaml +var ApplicationTemplate embed.FS + +//go:embed dockerfile-component.template.yaml +var DockerfileComponentTemplate embed.FS + +type Config struct { + OpenShiftReleasePath string + ApplicationName string + + Includes []string + Excludes []string + ExcludesImages []string + + ResourcesOutputPath string + + Nudges []string +} + +func Generate(cfg Config) error { + + if err := os.RemoveAll(cfg.ResourcesOutputPath); err != nil { + return fmt.Errorf("failed to remove %q directory: %w", cfg.ResourcesOutputPath, err) + } + + includes, err := toRegexp(cfg.Includes) + if err != nil { + return fmt.Errorf("failed to create regular expressions for %+v: %w", cfg.Includes, err) + } + excludes, err := toRegexp(cfg.Excludes) + if err != nil { + return fmt.Errorf("failed to create regular expressions for %+v: %w", cfg.Excludes, err) + } + excludeImages, err := toRegexp(cfg.ExcludesImages) + if err != nil { + return fmt.Errorf("failed to create regular expressions for %+v: %w", cfg.ExcludesImages, err) + } + + configs, err := collectConfigurations(cfg.OpenShiftReleasePath, includes, excludes) + if err != nil { + return err + } + + log.Printf("Found %d configs", len(configs)) + + funcs := template.FuncMap{ + "sanitize": sanitize, + "truncate": truncate, + } + + applicationTemplate, err := template.New("application.template.yaml").Funcs(funcs).ParseFS(ApplicationTemplate, "*.yaml") + if err != nil { + return fmt.Errorf("failed to parse application template: %w", err) + } + dockerfileComponentTemplate, err := template.New("dockerfile-component.template.yaml").Funcs(funcs).ParseFS(DockerfileComponentTemplate, "*.yaml") + if err != nil { + return fmt.Errorf("failed to parse dockerfile component template: %w", err) + } + + applications := make(map[string]map[string]DockerfileApplicationConfig, 8) + for _, c := range configs { + appKey := truncate(sanitize(cfg.ApplicationName)) + if _, ok := applications[appKey]; !ok { + applications[appKey] = make(map[string]DockerfileApplicationConfig, 8) + } + for _, ib := range c.Images { + + ignore := false + for _, r := range excludeImages { + if r.MatchString(string(ib.To)) { + ignore = true + break + } + } + if ignore { + continue + } + + applications[appKey][dockerfileComponentKey(c.ReleaseBuildConfiguration, ib)] = DockerfileApplicationConfig{ + ApplicationName: cfg.ApplicationName, + ReleaseBuildConfiguration: c.ReleaseBuildConfiguration, + Path: c.Path, + ProjectDirectoryImageBuildStepConfiguration: ib, + Nudges: cfg.Nudges, + } + } + } + + for appKey, components := range applications { + + for componentKey, config := range components { + buf := &bytes.Buffer{} + + appPath := filepath.Join(cfg.ResourcesOutputPath, "applications", appKey, fmt.Sprintf("%s.yaml", appKey)) + if err := os.MkdirAll(filepath.Dir(appPath), 0777); err != nil { + return fmt.Errorf("failed to create directory for %q: %w", appPath, err) + } + + if err := applicationTemplate.Execute(buf, config); err != nil { + return fmt.Errorf("failed to execute template for application %q: %w", appKey, err) + } + if err := os.WriteFile(appPath, buf.Bytes(), 0777); err != nil { + return fmt.Errorf("failed to write application file %q: %w", appPath, err) + } + + buf.Reset() + + componentPath := filepath.Join(cfg.ResourcesOutputPath, "applications", appKey, "components", fmt.Sprintf("%s.yaml", componentKey)) + if err := os.MkdirAll(filepath.Dir(componentPath), 0777); err != nil { + return fmt.Errorf("failed to create directory for %q: %w", componentPath, err) + } + + if err := dockerfileComponentTemplate.Execute(buf, config); err != nil { + return fmt.Errorf("failed to execute template for component %q: %w", componentKey, err) + } + if err := os.WriteFile(componentPath, buf.Bytes(), 0777); err != nil { + return fmt.Errorf("failed to write component file %q: %w", componentPath, err) + } + } + } + + return nil +} + +func collectConfigurations(openshiftReleasePath string, includes []*regexp.Regexp, excludes []*regexp.Regexp) ([]TemplateConfig, error) { + configs := make([]TemplateConfig, 0, 8) + err := filepath.WalkDir(openshiftReleasePath, func(path string, info fs.DirEntry, err error) error { + if info.IsDir() { + return nil + } + + matchablePath, err := filepath.Rel(openshiftReleasePath, path) + if err != nil { + return fmt.Errorf("failed to get relative path for %q (base path %q): %w", matchablePath, openshiftReleasePath, err) + } + + shouldInclude := false + for _, i := range includes { + if i.MatchString(matchablePath) { + shouldInclude = true + } + for _, x := range excludes { + if x.MatchString(matchablePath) { + shouldInclude = false + } + } + } + if !shouldInclude { + return nil + } + + log.Printf("Parsing file %q\n", path) + + c, err := parseConfig(path) + if err != nil { + return fmt.Errorf("failed to parse CI config in %q: %w", path, err) + } + + configs = append(configs, TemplateConfig{ + ReleaseBuildConfiguration: *c, + Path: path, + }) + + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed while walking directory %q: %w\n", openshiftReleasePath, err) + } + return configs, nil +} + +type TemplateConfig struct { + cioperatorapi.ReleaseBuildConfiguration + Path string +} + +type DockerfileApplicationConfig struct { + ApplicationName string + ReleaseBuildConfiguration cioperatorapi.ReleaseBuildConfiguration + Path string + ProjectDirectoryImageBuildStepConfiguration cioperatorapi.ProjectDirectoryImageBuildStepConfiguration + Nudges []string +} + +func parseConfig(path string) (*cioperatorapi.ReleaseBuildConfiguration, error) { + // Going directly from YAML raw input produces unexpected configs (due to missing YAML tags), + // so we convert YAML to JSON and unmarshal the struct from the JSON object. + y, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read file %q: %w", path, err) + } + j, err := gyaml.YAMLToJSON(y) + if err != nil { + return nil, fmt.Errorf("failed to convert YAML to JSON: %w", err) + } + + jobConfig := &cioperatorapi.ReleaseBuildConfiguration{} + if err := json.Unmarshal(j, jobConfig); err != nil { + return nil, fmt.Errorf("failed to unmarshal file %q into %T: %w", path, jobConfig, err) + } + + return jobConfig, err +} + +func toRegexp(rawRegexps []string) ([]*regexp.Regexp, error) { + regexps := make([]*regexp.Regexp, 0, len(rawRegexps)) + for _, i := range rawRegexps { + r, err := regexp.Compile(i) + if err != nil { + return regexps, fmt.Errorf("regex %q doesn't compile: %w", i, err) + } + regexps = append(regexps, r) + } + return regexps, nil +} + +func dockerfileComponentKey(cfg cioperatorapi.ReleaseBuildConfiguration, ib cioperatorapi.ProjectDirectoryImageBuildStepConfiguration) string { + return fmt.Sprintf("%s-%s-%s-%s", cfg.Metadata.Org, cfg.Metadata.Repo, cfg.Metadata.Branch, ib.To) +} + +func applicationKey(cfg cioperatorapi.ReleaseBuildConfiguration) string { + return fmt.Sprintf("%s-%s-%s", cfg.Metadata.Org, cfg.Metadata.Repo, cfg.Metadata.Branch) +} + +func sanitize(input interface{}) string { + in := fmt.Sprintf("%s", input) + // TODO very basic name sanitizer + return strings.ReplaceAll(strings.ReplaceAll(in, ".", ""), " ", "-") +} + +func truncate(input interface{}) string { + in := fmt.Sprintf("%s", input) + // TODO very basic name sanitizer + return Name(in, "") +} + +const ( + longest = 63 + md5Len = 32 + head = longest - md5Len // How much to truncate to fit the hash. +) + +// Name generates a name for the resource based upon the parent resource and suffix. +// If the concatenated name is longer than K8s permits the name is hashed and truncated to permit +// construction of the resource, but still keeps it unique. +// If the suffix itself is longer than 31 characters, then the whole string will be hashed +// and `parent|hash|suffix` will be returned, where parent and suffix will be trimmed to +// fit (prefix of parent at most of length 31, and prefix of suffix at most length 30). +func Name(parent, suffix string) string { + n := parent + if len(parent) > (longest - len(suffix)) { + // If the suffix is longer than the longest allowed suffix, then + // we hash the whole combined string and use that as the suffix. + if head-len(suffix) <= 0 { + //nolint:gosec // No strong cryptography needed. + h := md5.Sum([]byte(parent + suffix)) + // 1. trim parent, if needed + if head < len(parent) { + parent = parent[:head] + } + // Format the return string, if it's shorter than longest: pad with + // beginning of the suffix. This happens, for example, when parent is + // short, but the suffix is very long. + ret := parent + fmt.Sprintf("%x", h) + if d := longest - len(ret); d > 0 { + ret += suffix[:d] + } + return makeValidName(ret) + } + //nolint:gosec // No strong cryptography needed. + n = fmt.Sprintf("%s%x", parent[:head-len(suffix)], md5.Sum([]byte(parent))) + } + return n + suffix +} + +var isAlphanumeric = regexp.MustCompile(`^[a-zA-Z0-9]*$`) + +// If due to trimming above we're terminating the string with a non-alphanumeric +// character, remove it. +func makeValidName(n string) string { + for i := len(n) - 1; !isAlphanumeric.MatchString(string(n[i])); i-- { + n = n[:len(n)-1] + } + return n +} diff --git a/pkg/prowgen/prowgen.go b/pkg/prowgen/prowgen.go index 2a76c184..202008b0 100644 --- a/pkg/prowgen/prowgen.go +++ b/pkg/prowgen/prowgen.go @@ -126,9 +126,13 @@ func Main() { if err := RunOpenShiftReleaseGenerator(ctx, openShiftRelease); err != nil { log.Fatalln("Failed to run openshift/release generator after injecting Slack reporter", err) } - if err := PushBranch(ctx, openShiftRelease, remote, *branch, *inputConfig); err != nil { + if err := PushBranch(ctx, openShiftRelease, remote, *branch, "Sync Serverless CI "+*inputConfig); err != nil { log.Fatalln("Failed to push branch to openshift/release fork", *remote, err) } + + if err := GenerateKonflux(ctx, openShiftRelease, inConfig); err != nil { + log.Fatalln("Failed to generate Konflux configurations: %w", err) + } } func LoadConfig(path string) (*Config, error) { @@ -149,7 +153,7 @@ func LoadConfig(path string) (*Config, error) { return inConfig, nil } -func PushBranch(ctx context.Context, release Repository, remote *string, branch string, config string) error { +func PushBranch(ctx context.Context, release Repository, remote *string, branch string, commitMsg string) error { // Ignore error since remote and branch might be already there _, _ = run(ctx, release, "git", "checkout", "-b", branch) @@ -158,7 +162,7 @@ func PushBranch(ctx context.Context, release Repository, remote *string, branch if _, err := run(ctx, release, "git", "add", "."); err != nil { return err } - if _, err := run(ctx, release, "git", "commit", "-m", "Sync Serverless CI "+config); err != nil { + if _, err := run(ctx, release, "git", "commit", "-m", commitMsg); err != nil { // Ignore error since we could have nothing to commit log.Println("Ignored error", err) } diff --git a/pkg/prowgen/prowgen_config.go b/pkg/prowgen/prowgen_config.go index a613bf7d..0017de16 100644 --- a/pkg/prowgen/prowgen_config.go +++ b/pkg/prowgen/prowgen_config.go @@ -71,6 +71,13 @@ type Branch struct { OpenShiftVersions []OpenShift `json:"openShiftVersions,omitempty" yaml:"openShiftVersions,omitempty"` SkipE2EMatches []string `json:"skipE2EMatches,omitempty" yaml:"skipE2EMatches,omitempty"` SkipDockerFilesMatches []string `json:"skipDockerFilesMatches,omitempty" yaml:"skipDockerFilesMatches,omitempty"` + Konflux Konflux `json:"konflux,omitempty" yaml:"konflux,omitempty"` +} + +type Konflux struct { + Enabled bool + + Nudges []string } type OpenShift struct { diff --git a/pkg/prowgen/prowgen_konflux.go b/pkg/prowgen/prowgen_konflux.go new file mode 100644 index 00000000..1fd3d064 --- /dev/null +++ b/pkg/prowgen/prowgen_konflux.go @@ -0,0 +1,65 @@ +package prowgen + +import ( + "context" + "fmt" + + "github.com/openshift-knative/hack/pkg/konfluxgen" + "github.com/openshift-knative/hack/pkg/sobranch" +) + +const ( + KonfluxBranchPrefix = "sync-konflux/" +) + +func GenerateKonflux(ctx context.Context, openshiftRelease Repository, config *Config) error { + + for _, r := range config.Repositories { + for branchName, b := range config.Config.Branches { + if b.Konflux.Enabled { + + if err := GitMirror(ctx, r); err != nil { + return err + } + + if err := GitCheckout(ctx, r, branchName); err != nil { + return err + } + + cfg := konfluxgen.Config{ + OpenShiftReleasePath: openshiftRelease.RepositoryDirectory(), + ApplicationName: fmt.Sprintf("serverless-operator %s", sobranch.FromUpstreamVersion(branchName)), + Includes: []string{ + fmt.Sprintf("ci-operator/config/%s/.*%s.*.yaml", r.RepositoryDirectory(), branchName), + }, + Excludes: nil, + ExcludesImages: []string{ + ".*source.*", + ".*test.*", + }, + ResourcesOutputPath: fmt.Sprintf("%s/.konflux", r.RepositoryDirectory()), + Nudges: b.Konflux.Nudges, + } + + if err := konfluxgen.Generate(cfg); err != nil { + return fmt.Errorf("failed to generate Konflux configurations for %s (%s): %w", r.RepositoryDirectory(), branchName, err) + } + + // Special case "release-next" + targetBranch := branchName + if branchName == "release-next" { + targetBranch = "main" + } + + pushBranch := fmt.Sprintf("%s%s", KonfluxBranchPrefix, branchName) + commitMsg := fmt.Sprintf("[%s] Sync Konflux configurations", targetBranch) + + if err := PushBranch(ctx, r, nil, pushBranch, commitMsg); err != nil { + return err + } + } + } + } + + return nil +} diff --git a/pkg/sobranch/version.go b/pkg/sobranch/version.go index 8845d207..f9d8c83f 100644 --- a/pkg/sobranch/version.go +++ b/pkg/sobranch/version.go @@ -2,10 +2,21 @@ package sobranch import ( "fmt" + "strings" + "github.com/coreos/go-semver/semver" ) func FromUpstreamVersion(upstream string) string { + + upstream = strings.Replace(upstream, "release-v", "", 1) + upstream = strings.Replace(upstream, "release-", "", 1) + upstream = strings.Replace(upstream, "v", "", 1) + + dotParts := strings.SplitN(upstream, ".", 3) + if len(dotParts) == 2 { + upstream = upstream + ".0" + } soVersion := semver.New(upstream) for i := 0; i < 21; i++ { // Example 1.11 -> 1.32 soVersion.BumpMinor()