Skip to content

Commit

Permalink
Replay pipeline using cli exec by downloading metadata (#4103)
Browse files Browse the repository at this point in the history 
Co-authored-by: Anbraten <[email protected]>
  • Loading branch information
@6543 @anbraten
6543 and anbraten authored Sep 25, 2024
1 parent 1a6c8df commit fcc57df
Show file tree
Hide file tree
Showing 21 changed files with 927 additions and 194 deletions.
25 changes: 18 additions & 7 deletions cli/exec/exec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import (
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/kubernetes"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/local"
backend_types "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/compiler"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/linter"
Expand Down Expand Up @@ -76,6 +77,7 @@ func execDir(ctx context.Context, c *cli.Command, dir string) error {
if runtime.GOOS == "windows" {
repoPath = convertPathForWindows(repoPath)
}
// TODO: respect depends_on and do parallel runs with output to multiple _windows_ e.g. tmux like
return filepath.Walk(dir, func(path string, info os.FileInfo, e error) error {
if e != nil {
return e
Expand All @@ -84,7 +86,7 @@ func execDir(ctx context.Context, c *cli.Command, dir string) error {
// check if it is a regular file (not dir)
if info.Mode().IsRegular() && (strings.HasSuffix(info.Name(), ".yaml") || strings.HasSuffix(info.Name(), ".yml")) {
fmt.Println("#", info.Name())
_ = runExec(ctx, c, path, repoPath) // TODO: should we drop errors or store them and report back?
_ = runExec(ctx, c, path, repoPath, false) // TODO: should we drop errors or store them and report back?
fmt.Println("")
return nil
}
Expand All @@ -103,10 +105,10 @@ func execFile(ctx context.Context, c *cli.Command, file string) error {
if runtime.GOOS == "windows" {
repoPath = convertPathForWindows(repoPath)
}
return runExec(ctx, c, file, repoPath)
return runExec(ctx, c, file, repoPath, true)
}

func runExec(ctx context.Context, c *cli.Command, file, repoPath string) error {
func runExec(ctx context.Context, c *cli.Command, file, repoPath string, singleExec bool) error {
dat, err := os.ReadFile(file)
if err != nil {
return err
Expand All @@ -121,19 +123,28 @@ func runExec(ctx context.Context, c *cli.Command, file, repoPath string) error {
axes = append(axes, matrix.Axis{})
}
for _, axis := range axes {
err := execWithAxis(ctx, c, file, repoPath, axis)
err := execWithAxis(ctx, c, file, repoPath, axis, singleExec)
if err != nil {
return err
}
}
return nil
}

func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, axis matrix.Axis) error {
metadata, err := metadataFromContext(ctx, c, axis)
func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, axis matrix.Axis, singleExec bool) error {
var metadataWorkflow *metadata.Workflow
if !singleExec {
// TODO: proper try to use the engine to generate the same metadata for workflows
// https://github.com/woodpecker-ci/woodpecker/pull/3967
metadataWorkflow.Name = strings.TrimSuffix(strings.TrimSuffix(file, ".yaml"), ".yml")
}
metadata, err := metadataFromContext(ctx, c, axis, metadataWorkflow)
if err != nil {
return fmt.Errorf("could not create metadata: %w", err)
} else if metadata == nil {
return fmt.Errorf("metadata is nil")
}

environ := metadata.Environ()
var secrets []compiler.Secret
for key, val := range metadata.Workflow.Matrix {
Expand Down Expand Up @@ -239,7 +250,7 @@ func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, ax
c.String("netrc-password"),
c.String("netrc-machine"),
),
compiler.WithMetadata(metadata),
compiler.WithMetadata(*metadata),
compiler.WithSecret(secrets...),
compiler.WithEnviron(pipelineEnv),
).Compile(conf)
Expand Down
5 changes: 5 additions & 0 deletions cli/exec/flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ var flags = []cli.Flag{
Name: "repo-path",
Usage: "path to local repository",
},
&cli.StringFlag{
Sources: cli.EnvVars("WOODPECKER_METADATA_FILE"),
Name: "metadata-file",
Usage: "path to pipeline metadata file (normally downloaded from UI). Parameters can be adjusted by applying additional cli flags",
},
&cli.DurationFlag{
Sources: cli.EnvVars("WOODPECKER_TIMEOUT"),
Name: "timeout",
Expand Down
214 changes: 119 additions & 95 deletions cli/exec/metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"runtime"
"strings"

Expand All @@ -29,108 +30,131 @@ import (
)

// return the metadata from the cli context.
func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) (metadata.Metadata, error) {
func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis, w *metadata.Workflow) (*metadata.Metadata, error) {
m := &metadata.Metadata{}

if c.IsSet("metadata-file") {
metadataFile, err := os.Open(c.String("metadata-file"))
if err != nil {
return nil, err
}
defer metadataFile.Close()

if err := json.NewDecoder(metadataFile).Decode(m); err != nil {
return nil, err
}
}

platform := c.String("system-platform")
if platform == "" {
platform = runtime.GOOS + "/" + runtime.GOARCH
}

fullRepoName := c.String("repo-name")
repoOwner := ""
repoName := ""
if idx := strings.LastIndex(fullRepoName, "/"); idx != -1 {
repoOwner = fullRepoName[:idx]
repoName = fullRepoName[idx+1:]
}

var changedFiles []string
changedFilesRaw := c.String("pipeline-changed-files")
if len(changedFilesRaw) != 0 && changedFilesRaw[0] == '[' {
if err := json.Unmarshal([]byte(changedFilesRaw), &changedFiles); err != nil {
return metadata.Metadata{}, fmt.Errorf("pipeline-changed-files detected json but could not parse it: %w", err)
metadataFileAndOverrideOrDefault(c, "repo-name", func(fullRepoName string) {
if idx := strings.LastIndex(fullRepoName, "/"); idx != -1 {
m.Repo.Owner = fullRepoName[:idx]
m.Repo.Name = fullRepoName[idx+1:]
}
} else {
for _, file := range strings.Split(changedFilesRaw, ",") {
changedFiles = append(changedFiles, strings.TrimSpace(file))
}, c.String)

var err error
metadataFileAndOverrideOrDefault(c, "pipeline-changed-files", func(changedFilesRaw string) {
var changedFiles []string
if len(changedFilesRaw) != 0 && changedFilesRaw[0] == '[' {
if jsonErr := json.Unmarshal([]byte(changedFilesRaw), &changedFiles); jsonErr != nil {
err = fmt.Errorf("pipeline-changed-files detected json but could not parse it: %w", jsonErr)
}
} else {
for _, file := range strings.Split(changedFilesRaw, ",") {
changedFiles = append(changedFiles, strings.TrimSpace(file))
}
}
m.Curr.Commit.ChangedFiles = changedFiles
}, c.String)
if err != nil {
return nil, err
}

return metadata.Metadata{
Repo: metadata.Repo{
Name: repoName,
Owner: repoOwner,
RemoteID: c.String("repo-remote-id"),
ForgeURL: c.String("repo-url"),
SCM: c.String("repo-scm"),
Branch: c.String("repo-default-branch"),
CloneURL: c.String("repo-clone-url"),
CloneSSHURL: c.String("repo-clone-ssh-url"),
Private: c.Bool("repo-private"),
Trusted: c.Bool("repo-trusted"),
},
Curr: metadata.Pipeline{
Number: c.Int("pipeline-number"),
Parent: c.Int("pipeline-parent"),
Created: c.Int("pipeline-created"),
Started: c.Int("pipeline-started"),
Finished: c.Int("pipeline-finished"),
Status: c.String("pipeline-status"),
Event: c.String("pipeline-event"),
ForgeURL: c.String("pipeline-url"),
DeployTo: c.String("pipeline-deploy-to"),
DeployTask: c.String("pipeline-deploy-task"),
Commit: metadata.Commit{
Sha: c.String("commit-sha"),
Ref: c.String("commit-ref"),
Refspec: c.String("commit-refspec"),
Branch: c.String("commit-branch"),
Message: c.String("commit-message"),
Author: metadata.Author{
Name: c.String("commit-author-name"),
Email: c.String("commit-author-email"),
Avatar: c.String("commit-author-avatar"),
},
PullRequestLabels: c.StringSlice("commit-pull-labels"),
IsPrerelease: c.Bool("commit-release-is-pre"),
ChangedFiles: changedFiles,
},
},
Prev: metadata.Pipeline{
Number: c.Int("prev-pipeline-number"),
Created: c.Int("prev-pipeline-created"),
Started: c.Int("prev-pipeline-started"),
Finished: c.Int("prev-pipeline-finished"),
Status: c.String("prev-pipeline-status"),
Event: c.String("prev-pipeline-event"),
ForgeURL: c.String("prev-pipeline-url"),
Commit: metadata.Commit{
Sha: c.String("prev-commit-sha"),
Ref: c.String("prev-commit-ref"),
Refspec: c.String("prev-commit-refspec"),
Branch: c.String("prev-commit-branch"),
Message: c.String("prev-commit-message"),
Author: metadata.Author{
Name: c.String("prev-commit-author-name"),
Email: c.String("prev-commit-author-email"),
Avatar: c.String("prev-commit-author-avatar"),
},
},
},
Workflow: metadata.Workflow{
Name: c.String("workflow-name"),
Number: int(c.Int("workflow-number")),
Matrix: axis,
},
Sys: metadata.System{
Name: c.String("system-name"),
URL: c.String("system-url"),
Host: c.String("system-host"),
Platform: platform,
Version: version.Version,
},
Forge: metadata.Forge{
Type: c.String("forge-type"),
URL: c.String("forge-url"),
},
}, nil
// Repo
metadataFileAndOverrideOrDefault(c, "repo-remote-id", func(s string) { m.Repo.RemoteID = s }, c.String)
metadataFileAndOverrideOrDefault(c, "repo-url", func(s string) { m.Repo.ForgeURL = s }, c.String)
metadataFileAndOverrideOrDefault(c, "repo-scm", func(s string) { m.Repo.SCM = s }, c.String)
metadataFileAndOverrideOrDefault(c, "repo-default-branch", func(s string) { m.Repo.Branch = s }, c.String)
metadataFileAndOverrideOrDefault(c, "repo-clone-url", func(s string) { m.Repo.CloneURL = s }, c.String)
metadataFileAndOverrideOrDefault(c, "repo-clone-ssh-url", func(s string) { m.Repo.CloneSSHURL = s }, c.String)
metadataFileAndOverrideOrDefault(c, "repo-private", func(b bool) { m.Repo.Private = b }, c.Bool)
metadataFileAndOverrideOrDefault(c, "repo-trusted", func(b bool) { m.Repo.Trusted = b }, c.Bool)

// Current Pipeline
metadataFileAndOverrideOrDefault(c, "pipeline-number", func(i int64) { m.Curr.Number = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "pipeline-parent", func(i int64) { m.Curr.Parent = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "pipeline-created", func(i int64) { m.Curr.Created = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "pipeline-started", func(i int64) { m.Curr.Started = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "pipeline-finished", func(i int64) { m.Curr.Finished = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "pipeline-status", func(s string) { m.Curr.Status = s }, c.String)
metadataFileAndOverrideOrDefault(c, "pipeline-event", func(s string) { m.Curr.Event = s }, c.String)
metadataFileAndOverrideOrDefault(c, "pipeline-url", func(s string) { m.Curr.ForgeURL = s }, c.String)
metadataFileAndOverrideOrDefault(c, "pipeline-deploy-to", func(s string) { m.Curr.DeployTo = s }, c.String)
metadataFileAndOverrideOrDefault(c, "pipeline-deploy-task", func(s string) { m.Curr.DeployTask = s }, c.String)

// Current Pipeline Commit
metadataFileAndOverrideOrDefault(c, "commit-sha", func(s string) { m.Curr.Commit.Sha = s }, c.String)
metadataFileAndOverrideOrDefault(c, "commit-ref", func(s string) { m.Curr.Commit.Ref = s }, c.String)
metadataFileAndOverrideOrDefault(c, "commit-refspec", func(s string) { m.Curr.Commit.Refspec = s }, c.String)
metadataFileAndOverrideOrDefault(c, "commit-branch", func(s string) { m.Curr.Commit.Branch = s }, c.String)
metadataFileAndOverrideOrDefault(c, "commit-message", func(s string) { m.Curr.Commit.Message = s }, c.String)
metadataFileAndOverrideOrDefault(c, "commit-author-name", func(s string) { m.Curr.Commit.Author.Name = s }, c.String)
metadataFileAndOverrideOrDefault(c, "commit-author-email", func(s string) { m.Curr.Commit.Author.Email = s }, c.String)
metadataFileAndOverrideOrDefault(c, "commit-author-avatar", func(s string) { m.Curr.Commit.Author.Avatar = s }, c.String)

metadataFileAndOverrideOrDefault(c, "commit-pull-labels", func(sl []string) { m.Curr.Commit.PullRequestLabels = sl }, c.StringSlice)
metadataFileAndOverrideOrDefault(c, "commit-release-is-pre", func(b bool) { m.Curr.Commit.IsPrerelease = b }, c.Bool)

// Previous Pipeline
metadataFileAndOverrideOrDefault(c, "prev-pipeline-number", func(i int64) { m.Prev.Number = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "prev-pipeline-created", func(i int64) { m.Prev.Created = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "prev-pipeline-started", func(i int64) { m.Prev.Started = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "prev-pipeline-finished", func(i int64) { m.Prev.Finished = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "prev-pipeline-status", func(s string) { m.Prev.Status = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-pipeline-event", func(s string) { m.Prev.Event = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-pipeline-url", func(s string) { m.Prev.ForgeURL = s }, c.String)

// Previous Pipeline Commit
metadataFileAndOverrideOrDefault(c, "prev-commit-sha", func(s string) { m.Prev.Commit.Sha = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-commit-ref", func(s string) { m.Prev.Commit.Ref = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-commit-refspec", func(s string) { m.Prev.Commit.Refspec = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-commit-branch", func(s string) { m.Prev.Commit.Branch = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-commit-message", func(s string) { m.Prev.Commit.Message = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-commit-author-name", func(s string) { m.Prev.Commit.Author.Name = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-commit-author-email", func(s string) { m.Prev.Commit.Author.Email = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-commit-author-avatar", func(s string) { m.Prev.Commit.Author.Avatar = s }, c.String)

// Workflow
metadataFileAndOverrideOrDefault(c, "workflow-name", func(s string) { m.Workflow.Name = s }, c.String)
metadataFileAndOverrideOrDefault(c, "workflow-number", func(i int64) { m.Workflow.Number = int(i) }, c.Int)
m.Workflow.Matrix = axis

// System
metadataFileAndOverrideOrDefault(c, "system-name", func(s string) { m.Sys.Name = s }, c.String)
metadataFileAndOverrideOrDefault(c, "system-url", func(s string) { m.Sys.URL = s }, c.String)
metadataFileAndOverrideOrDefault(c, "system-host", func(s string) { m.Sys.Host = s }, c.String)
m.Sys.Platform = platform
m.Sys.Version = version.Version

// Forge
metadataFileAndOverrideOrDefault(c, "forge-type", func(s string) { m.Forge.Type = s }, c.String)
metadataFileAndOverrideOrDefault(c, "forge-url", func(s string) { m.Forge.URL = s }, c.String)

if w != nil {
m.Workflow = *w
}

return m, nil
}

// metadataFileAndOverrideOrDefault will either use the flag default or if metadata file is set only overload if explicit set.
func metadataFileAndOverrideOrDefault[T any](c *cli.Command, flag string, setter func(T), getter func(string) T) {
if !c.IsSet("metadata-file") || c.IsSet(flag) {
setter(getter(flag))
}
}
Loading

0 comments on commit fcc57df

Please sign in to comment.